diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java index 63937f8..75277c4 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Stage.java @@ -152,7 +152,7 @@ public void extractVertex(JSONObject object) throws Exception { } // The opTree in vertex is extracted for (Vertex v : vertexs.values()) { - if (v.vertexType == VertexType.MAP || v.vertexType == VertexType.REDUCE) { + if (v.vertexType == VertexType.VERTEX) { v.extractOpTree(); v.checkMultiReduceOperator(); } diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java index 3d559bd..3ebd060 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java @@ -63,12 +63,12 @@ public String tag; public static enum VertexType { - MAP, REDUCE, UNION, UNKNOWN + VERTEX, UNION, UNKNOWN }; public VertexType vertexType; public static enum EdgeType { - BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, UNKNOWN + BROADCAST, PARTITION_SORT, CO_PARTITION, PARTITION, UNKNOWN }; public EdgeType edgeType; @@ -76,10 +76,8 @@ public Vertex(String name, JSONObject vertexObject, TezJsonParser tezJsonParser) super(); this.name = name; if (this.name != null) { - if (this.name.contains("Map")) { - this.vertexType = VertexType.MAP; - } else if (this.name.contains("Reduce")) { - this.vertexType = VertexType.REDUCE; + if (this.name.contains("Vertex")) { + this.vertexType = VertexType.VERTEX; } else if (this.name.contains("Union")) { this.vertexType = VertexType.UNION; } else { @@ -273,7 +271,7 @@ public void print(Printer printer, int indentFlag, String type, Vertex callingVe */ public void checkMultiReduceOperator() { // check if it is a reduce vertex and its children is more than 1; - if (!this.name.contains("Reduce") || this.rootOps.size() < 2) { + if (!this.name.contains("Vertex") || this.rootOps.size() < 2) { return; } // check if all the child ops are reduce output operators @@ -290,14 +288,14 @@ public void setType(String type) { case "BROADCAST_EDGE": this.edgeType = EdgeType.BROADCAST; break; - case "SIMPLE_EDGE": - this.edgeType = EdgeType.SHUFFLE; + case "SORT_PARTITION_EDGE": + this.edgeType = EdgeType.PARTITION_SORT; break; - case "CUSTOM_SIMPLE_EDGE": - this.edgeType = EdgeType.PARTITION_ONLY_SHUFFLE; + case "PARTITION_EDGE": + this.edgeType = EdgeType.PARTITION; break; - case "CUSTOM_EDGE": - this.edgeType = EdgeType.MULTICAST; + case "CO_PARTITION_EDGE": + this.edgeType = EdgeType.CO_PARTITION; break; default: this.edgeType = EdgeType.UNKNOWN; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 12a03d0..1570fbf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -224,8 +224,8 @@ public static final String HIVE_ADDED_JARS = "hive.added.jars"; public static final String VECTOR_MODE = "VECTOR_MODE"; public static final String USE_VECTORIZED_INPUT_FILE_FORMAT = "USE_VECTORIZED_INPUT_FILE_FORMAT"; - public static String MAPNAME = "Map "; - public static String REDUCENAME = "Reducer "; + public static String MAPNAME = "Vertex "; + public static String REDUCENAME = "Vertex "; /** * ReduceField: diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index aa2dfc7..eadcf93 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -273,7 +273,7 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, case BROADCAST_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; - case CUSTOM_EDGE: { + case CO_PARTITION_EDGE: { mergeInputClass = ConcatenatedMergedKeyValueInput.class; int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); @@ -288,11 +288,11 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, break; } - case CUSTOM_SIMPLE_EDGE: + case PARTITION_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; - case SIMPLE_EDGE: + case SORT_PARTITION_EDGE: setupAutoReducerParallelism(edgeProp, w); // fall through @@ -319,7 +319,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr throws IOException { switch(edgeProp.getEdgeType()) { - case CUSTOM_EDGE: { + case CO_PARTITION_EDGE: { int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); DataOutputBuffer dob = new DataOutputBuffer(); @@ -332,7 +332,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr w.setVertexManagerPlugin(desc); break; } - case SIMPLE_EDGE: { + case SORT_PARTITION_EDGE: { setupAutoReducerParallelism(edgeProp, w); break; } @@ -364,7 +364,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et1Conf.createDefaultBroadcastEdgeProperty(); - case CUSTOM_EDGE: + case CO_PARTITION_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et2Conf = UnorderedPartitionedKVEdgeConfig @@ -382,7 +382,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); - case CUSTOM_SIMPLE_EDGE: + case PARTITION_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig @@ -392,7 +392,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et3Conf.createDefaultEdgeProperty(); - case SIMPLE_EDGE: + case SORT_PARTITION_EDGE: default: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java index 3a6baca..54bd63c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java @@ -252,10 +252,10 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma ReduceSinkOperator.class); if (rootOp == null) { // likely we found a table scan operator - edgeType = EdgeType.CUSTOM_EDGE; + edgeType = EdgeType.CO_PARTITION_EDGE; } else { // we have found a reduce sink - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } } else { Operator rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted( @@ -263,16 +263,16 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma TableScanOperator.class); if (rootOp != null) { // likely we found a table scan operator - edgeType = EdgeType.CUSTOM_EDGE; + edgeType = EdgeType.CO_PARTITION_EDGE; } else { // we have found a reduce sink - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } } } else if (mapJoinOp.getConf().isDynamicPartitionHashJoin()) { - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } - if (edgeType == EdgeType.CUSTOM_EDGE) { + if (edgeType == EdgeType.CO_PARTITION_EDGE) { // disable auto parallelism for bucket map joins parentRS.getConf().setReducerTraits(EnumSet.of(FIXED)); } @@ -284,7 +284,7 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma TezWork tezWork = context.currentTask.getWork(); LOG.debug("connecting "+parentWork.getName()+" with "+myWork.getName()); tezWork.connect(parentWork, myWork, edgeProp); - if (edgeType == EdgeType.CUSTOM_EDGE) { + if (edgeType == EdgeType.CO_PARTITION_EDGE) { tezWork.setVertexType(myWork, VertexType.INITIALIZED_EDGES); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index aee74ad..a640e81 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -500,15 +500,15 @@ public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork follo if (reduceWork.getReducer() instanceof MapJoinOperator) { MapJoinOperator joinOp = (MapJoinOperator) reduceWork.getReducer(); if (joinOp.getConf().isDynamicPartitionHashJoin()) { - return EdgeType.CUSTOM_SIMPLE_EDGE; + return EdgeType.PARTITION_EDGE; } } } if(!reduceSinkOperator.getConf().isOrdering()) { //if no sort keys are specified, use an edge that does not sort - return EdgeType.CUSTOM_SIMPLE_EDGE; + return EdgeType.PARTITION_EDGE; } - return EdgeType.SIMPLE_EDGE; + return EdgeType.SORT_PARTITION_EDGE; } public static void processDynamicMinMaxPushDownOperator( diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java index 97f3300..6e440ea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java @@ -283,7 +283,7 @@ public Object process(Node nd, Stack stack, LOG.debug("connecting "+parentWork.getName()+" with "+work.getName()); TezEdgeProperty edgeProp = parentWorkMap.getValue(); tezWork.connect(parentWork, work, edgeProp); - if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) { + if (edgeProp.getEdgeType() == EdgeType.CO_PARTITION_EDGE) { tezWork.setVertexType(work, VertexType.INITIALIZED_EDGES); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java index a3aa12f..21ae21e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java @@ -23,11 +23,11 @@ public class TezEdgeProperty { public enum EdgeType { - SIMPLE_EDGE, + SORT_PARTITION_EDGE, BROADCAST_EDGE, CONTAINS, - CUSTOM_EDGE, - CUSTOM_SIMPLE_EDGE, + CO_PARTITION_EDGE, + PARTITION_EDGE, } private HiveConf hiveConf; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index 5c012f3..c76c077 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -153,7 +153,7 @@ public Edge answer(InvocationOnMock invocation) throws Throwable { rws[0].setReducer(op); rws[1].setReducer(op); - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(mws[0], rws[0], edgeProp); work.connect(mws[1], rws[0], edgeProp); work.connect(rws[0], rws[1], edgeProp); diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java index c1e192c..ea72cb1 100644 --- ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java +++ ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java @@ -67,7 +67,7 @@ public void testConnect() throws Exception { BaseWork parent = nodes.get(0); BaseWork child = nodes.get(1); - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, child, edgeProp); Assert.assertEquals(work.getParents(child).size(), 1); @@ -84,7 +84,7 @@ public void testConnect() throws Exception { Assert.assertEquals(work.getChildren(w).size(), 0); } - Assert.assertEquals(work.getEdgeProperty(parent, child).getEdgeType(), EdgeType.SIMPLE_EDGE); + Assert.assertEquals(work.getEdgeProperty(parent, child).getEdgeType(), EdgeType.SORT_PARTITION_EDGE); } @Test @@ -117,7 +117,7 @@ public void testDisconnect() throws Exception { BaseWork parent = nodes.get(0); BaseWork children[] = {nodes.get(1), nodes.get(2)}; - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, children[0], edgeProp); work.connect(parent, children[1], edgeProp); @@ -136,7 +136,7 @@ public void testRemove() throws Exception { BaseWork parent = nodes.get(0); BaseWork children[] = {nodes.get(1), nodes.get(2)}; - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, children[0], edgeProp); work.connect(parent, children[1], edgeProp); @@ -151,7 +151,7 @@ public void testRemove() throws Exception { @Test public void testGetAllWork() throws Exception { - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); for (int i = 4; i > 0; --i) { work.connect(nodes.get(i), nodes.get(i-1), edgeProp); } diff --git ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out index d05bf64..cac6194 100644 --- ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out +++ ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acidtbldefault diff --git ql/src/test/results/clientpositive/llap/auto_join0.q.out ql/src/test/results/clientpositive/llap/auto_join0.q.out index cba6001..32d9461 100644 --- ql/src/test/results/clientpositive/llap/auto_join0.q.out +++ ql/src/test/results/clientpositive/llap/auto_join0.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( @@ -30,13 +30,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -54,25 +54,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -87,13 +69,13 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Reducer 6 + 1 Vertex 6 Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -109,7 +91,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -124,7 +106,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -142,7 +142,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( SELECT src1.key as k1, src1.value as v1, diff --git ql/src/test/results/clientpositive/llap/auto_join1.q.out ql/src/test/results/clientpositive/llap/auto_join1.q.out index 6a0a1d5..379c087 100644 --- ql/src/test/results/clientpositive/llap/auto_join1.q.out +++ ql/src/test/results/clientpositive/llap/auto_join1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -47,7 +47,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -67,7 +67,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) diff --git ql/src/test/results/clientpositive/llap/auto_join21.q.out ql/src/test/results/clientpositive/llap/auto_join21.q.out index 97b36f7..1d5f09e 100644 --- ql/src/test/results/clientpositive/llap/auto_join21.q.out +++ ql/src/test/results/clientpositive/llap/auto_join21.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -30,36 +30,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -84,7 +55,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,6 +69,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_join29.q.out ql/src/test/results/clientpositive/llap/auto_join29.q.out index a3cc39d..44a75b9 100644 --- ql/src/test/results/clientpositive/llap/auto_join29.q.out +++ ql/src/test/results/clientpositive/llap/auto_join29.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -30,36 +30,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -84,7 +55,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,6 +69,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -628,11 +628,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -645,39 +645,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -702,7 +670,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 322 Data size: 171948 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -716,6 +684,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1246,11 +1246,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1266,36 +1266,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1320,7 +1291,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 974 Data size: 520116 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1334,6 +1305,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1876,11 +1876,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1896,33 +1896,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1947,7 +1921,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1961,6 +1935,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2503,11 +2503,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2523,39 +2523,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2576,7 +2544,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2590,6 +2558,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2620,11 +2620,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2640,36 +2640,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2694,7 +2665,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2708,6 +2679,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3238,11 +3238,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3261,8 +3261,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3274,7 +3274,21 @@ STAGE PLANS: Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -3290,7 +3304,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3306,20 +3320,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3350,11 +3350,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3370,7 +3370,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -3392,8 +3392,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3405,7 +3405,21 @@ STAGE PLANS: Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3421,20 +3435,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3487,11 +3487,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3509,8 +3509,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3522,7 +3522,21 @@ STAGE PLANS: Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -3538,7 +3552,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3554,20 +3568,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_join30.q.out ql/src/test/results/clientpositive/llap/auto_join30.q.out index a26db55..bbfd9d7 100644 --- ql/src/test/results/clientpositive/llap/auto_join30.q.out +++ ql/src/test/results/clientpositive/llap/auto_join30.q.out @@ -23,12 +23,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -46,7 +46,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -65,19 +77,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -92,7 +92,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -103,7 +103,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -169,12 +169,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -189,23 +189,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -220,7 +204,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -231,7 +215,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -246,7 +230,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -310,12 +310,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -330,7 +330,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -346,19 +358,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -373,7 +373,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -384,7 +384,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -456,13 +456,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -480,26 +480,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -518,19 +511,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -547,8 +528,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 - 2 Reducer 7 + 0 Vertex 2 + 2 Vertex 7 Statistics: Num rows: 2974 Data size: 529372 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -559,7 +540,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -574,7 +555,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -649,14 +649,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -671,39 +671,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -715,7 +683,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -737,7 +705,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -752,7 +720,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -765,7 +749,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -840,14 +840,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -862,39 +862,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -906,7 +874,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -928,7 +896,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -943,7 +911,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -956,7 +940,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1031,14 +1031,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1053,39 +1053,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1097,7 +1065,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1119,7 +1087,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1134,7 +1102,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1147,7 +1131,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1222,14 +1222,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1244,39 +1244,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1288,7 +1256,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1310,7 +1278,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1325,7 +1293,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1338,7 +1322,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/auto_join_filters.q.out ql/src/test/results/clientpositive/llap/auto_join_filters.q.out index d1d9408..f4c1e79 100644 --- ql/src/test/results/clientpositive/llap/auto_join_filters.q.out +++ ql/src/test/results/clientpositive/llap/auto_join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -44,7 +44,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table sm POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -310,7 +310,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -320,7 +320,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -330,7 +330,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out index 5984e8f..75254df 100644 --- ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out +++ ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index a867bd2..a180b0d 100644 --- ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -100,7 +100,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -166,11 +166,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -212,7 +212,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,7 +231,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -321,12 +321,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -370,7 +370,44 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: b @@ -414,44 +451,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -540,10 +540,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -583,7 +583,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -657,10 +657,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -700,7 +700,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -841,7 +841,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -927,10 +927,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -970,7 +970,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1034,11 +1034,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1057,26 +1057,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key + 1) is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (key + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1095,7 +1076,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1110,6 +1091,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (key + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1155,10 +1155,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1198,7 +1198,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1264,10 +1264,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1320,7 +1320,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1402,10 +1402,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1445,7 +1445,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1541,7 +1541,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1755,10 +1755,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1814,7 +1814,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out index 4b646dc..318542a 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out @@ -112,11 +112,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -194,7 +194,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -211,12 +211,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -337,7 +337,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -405,11 +405,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -426,12 +426,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Vertex 3 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -552,7 +552,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -630,37 +661,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -698,11 +698,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -719,12 +719,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Vertex 3 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -845,7 +845,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -923,37 +954,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out index ab94c37..dd6bf84 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out @@ -71,12 +71,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -95,7 +97,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -108,7 +110,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -127,7 +144,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -140,7 +157,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -164,23 +181,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -240,11 +240,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -264,7 +264,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -283,7 +283,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -296,7 +296,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out index 7e3fe4c..0fca3d9 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out @@ -110,11 +110,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -191,7 +191,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -208,12 +208,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -332,7 +332,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -400,11 +400,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -481,7 +481,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -498,12 +498,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -622,7 +622,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -690,11 +690,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -767,7 +767,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -780,12 +780,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 key (type: string) 1 key (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -904,7 +904,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -972,11 +972,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1049,7 +1049,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1063,14 +1063,14 @@ STAGE PLANS: condition map: Inner Join 0 to 1 Inner Join 0 to 2 - Estimated key counts: Map 1 => 1, Map 4 => 116 + Estimated key counts: Vertex 1 => 1, Vertex 4 => 116 keys: 0 key (type: string) 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Position of Big Table: 1 Statistics: Num rows: 255 Data size: 27614 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1189,7 +1189,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Map 4 + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1311,37 +1342,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [c] /bucket_big/ds=2008-04-09 [c] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out index 054cf1a..c2b611f 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out @@ -134,7 +134,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -148,11 +148,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE), Vertex 2 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -230,7 +230,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -308,7 +308,7 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [b] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -326,25 +326,25 @@ STAGE PLANS: condition map: Inner Join 0 to 1 Inner Join 1 to 2 - Estimated key counts: Map 1 => 1, Map 2 => 1 + Estimated key counts: Vertex 1 => 1, Vertex 2 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) 2 _col0 (type: string) input vertices: - 0 Map 1 - 1 Map 2 + 0 Vertex 1 + 1 Vertex 2 Position of Big Table: 2 Statistics: Num rows: 255 Data size: 27614 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 5 => 1 + Estimated key counts: Vertex 5 => 1 keys: 0 1 input vertices: - 1 Map 5 + 1 Vertex 5 Position of Big Table: 0 Statistics: Num rows: 255 Data size: 29909 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -465,7 +465,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [c] /bucket_big/ds=2008-04-09 [c] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 5 Map Operator Tree: TableScan alias: d @@ -535,37 +566,6 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [d] - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -573,7 +573,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out index 9c37848..ba800b7 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out @@ -79,10 +79,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -102,7 +102,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -130,7 +130,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -295,10 +295,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -318,7 +318,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -346,7 +346,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -511,10 +511,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -534,7 +534,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -562,7 +562,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out index c618531..7dfb541 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out @@ -49,11 +49,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -69,7 +69,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -82,7 +82,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -98,21 +113,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -156,11 +156,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -176,7 +176,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -192,7 +192,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 207 Data size: 2080 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out index 286939b..17c85ab 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out @@ -49,11 +49,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -69,7 +69,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -82,7 +82,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -98,21 +113,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -135,11 +135,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -155,7 +155,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -171,7 +171,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -184,7 +184,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out index 42e4cb7..ac23dd5 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out @@ -94,11 +94,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -115,12 +115,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -241,7 +241,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -319,37 +350,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,12 +408,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -534,7 +534,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -612,37 +643,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out index 64c10ce..6ae25ab 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out @@ -94,11 +94,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -226,7 +226,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -243,12 +243,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -319,7 +319,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,12 +408,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -484,7 +484,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -612,37 +643,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -680,11 +680,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -701,12 +701,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -777,7 +777,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -905,37 +936,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out index 96854a9..e62cb2e 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out @@ -110,11 +110,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -242,7 +242,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -259,12 +259,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 4 + Estimated key counts: Vertex 1 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -335,7 +335,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -403,11 +403,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -424,12 +424,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -500,7 +500,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -628,37 +659,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -696,11 +696,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -717,12 +717,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -793,7 +793,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -921,37 +952,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out index b77a879..8ecb45f 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out @@ -75,10 +75,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -230,7 +230,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -292,10 +292,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -447,7 +447,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [a] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -509,11 +509,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -530,12 +530,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -605,7 +605,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -682,37 +713,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out index 9e60536..35245e2 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out @@ -83,11 +83,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -124,26 +124,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -162,7 +143,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -177,6 +158,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -210,11 +210,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -251,26 +251,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: d - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -289,7 +270,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -304,6 +285,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -337,11 +337,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -378,26 +378,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: h - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -416,7 +397,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -431,6 +412,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: h + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -464,11 +464,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -505,26 +505,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -543,7 +524,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -558,6 +539,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -591,10 +591,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -647,7 +647,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -695,11 +695,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -736,26 +736,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -774,7 +755,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -789,6 +770,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -822,11 +822,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -863,26 +863,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -901,7 +882,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -916,6 +897,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -949,11 +949,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -990,26 +990,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1028,7 +1009,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1043,6 +1024,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1076,10 +1076,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1132,7 +1132,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1180,11 +1180,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1221,26 +1221,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1259,7 +1240,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1274,6 +1255,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out index 13d3acc..d861a94 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out @@ -127,11 +127,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -276,12 +276,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 4 + Estimated key counts: Vertex 1 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -402,7 +402,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -472,11 +472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -493,12 +493,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -619,7 +619,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -747,37 +778,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -817,11 +817,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -838,12 +838,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -964,7 +964,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1092,37 +1123,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out index 7fc90fd..9a96912 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out @@ -127,11 +127,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -276,12 +276,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -402,7 +402,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -472,11 +472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -493,12 +493,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -619,7 +619,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -747,37 +778,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -817,11 +817,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -838,12 +838,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -964,7 +964,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1092,37 +1123,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out index c08a534..3768165 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out @@ -57,11 +57,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -80,7 +80,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -93,7 +93,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -112,21 +127,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -172,11 +172,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -196,7 +196,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -212,7 +212,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -231,22 +247,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -307,12 +307,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -332,7 +332,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) @@ -346,26 +346,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -384,7 +365,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -399,6 +380,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -474,13 +474,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -500,7 +500,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -516,7 +516,37 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -535,7 +565,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -555,7 +585,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -571,7 +601,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: b @@ -590,51 +635,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Reducer 5 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -709,11 +709,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -732,7 +732,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -745,7 +745,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -764,21 +779,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -838,11 +838,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -861,7 +861,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -874,7 +874,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -893,21 +908,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -991,11 +991,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1014,7 +1014,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1027,7 +1027,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1046,21 +1061,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1132,11 +1132,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1155,7 +1155,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1168,7 +1168,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1187,21 +1202,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1251,11 +1251,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1274,7 +1274,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1287,7 +1287,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1306,21 +1321,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1366,11 +1366,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1389,7 +1389,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1402,7 +1402,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1421,21 +1436,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1479,11 +1479,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1502,7 +1502,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1515,7 +1515,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1534,21 +1549,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1602,11 +1602,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1627,8 +1627,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1641,7 +1641,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1660,7 +1675,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -1679,21 +1694,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1761,11 +1761,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1784,7 +1784,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1797,7 +1797,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1816,21 +1831,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1888,11 +1888,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1911,7 +1911,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1924,7 +1924,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1943,21 +1958,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2003,11 +2003,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2027,7 +2027,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2043,7 +2043,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2062,22 +2078,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2138,12 +2138,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2163,7 +2163,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) @@ -2177,26 +2177,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2215,7 +2196,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2230,6 +2211,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2305,13 +2305,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2331,7 +2331,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2347,7 +2347,37 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2366,7 +2396,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -2386,72 +2416,23 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Reducer 5 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2466,6 +2447,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2540,11 +2540,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2563,7 +2563,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2576,7 +2576,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -2595,21 +2610,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2669,11 +2669,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2692,7 +2692,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2705,7 +2705,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2724,21 +2739,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2822,11 +2822,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2845,7 +2845,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2858,7 +2858,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -2877,21 +2892,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2963,11 +2963,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2986,7 +2986,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2999,7 +2999,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3018,21 +3033,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3078,11 +3078,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3101,7 +3101,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3114,7 +3114,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3133,21 +3148,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3191,11 +3191,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3214,7 +3214,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3227,7 +3227,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3246,21 +3261,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3314,11 +3314,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3339,8 +3339,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3353,7 +3353,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3372,7 +3387,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -3391,21 +3406,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3473,11 +3473,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3496,7 +3496,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3509,7 +3509,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3528,21 +3543,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucket2.q.out ql/src/test/results/clientpositive/llap/bucket2.q.out index be886b0..dc5db82 100644 --- ql/src/test/results/clientpositive/llap/bucket2.q.out +++ ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket3.q.out ql/src/test/results/clientpositive/llap/bucket3.q.out index 8e6d85c..3279500 100644 --- ql/src/test/results/clientpositive/llap/bucket3.q.out +++ ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket4.q.out ql/src/test/results/clientpositive/llap/bucket4.q.out index b4f7ca1..88445a2 100644 --- ql/src/test/results/clientpositive/llap/bucket4.q.out +++ ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -100,7 +100,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket5.q.out ql/src/test/results/clientpositive/llap/bucket5.q.out index ef5067b..f5647e6 100644 --- ql/src/test/results/clientpositive/llap/bucket5.q.out +++ ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -42,11 +42,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -131,7 +131,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -173,7 +173,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket6.q.out ql/src/test/results/clientpositive/llap/bucket6.q.out index 20895f8..207ed7f 100644 --- ql/src/test/results/clientpositive/llap/bucket6.q.out +++ ql/src/test/results/clientpositive/llap/bucket6.q.out @@ -23,36 +23,36 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/llap/bucket_groupby.q.out ql/src/test/results/clientpositive/llap/bucket_groupby.q.out index 2c3cf2f..01618ef 100644 --- ql/src/test/results/clientpositive/llap/bucket_groupby.q.out +++ ql/src/test/results/clientpositive/llap/bucket_groupby.q.out @@ -55,11 +55,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -83,7 +83,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -98,7 +98,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -183,11 +183,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -211,7 +211,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -226,7 +226,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -285,10 +285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -312,7 +312,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -366,10 +366,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -393,7 +393,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -447,11 +447,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -475,7 +475,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -490,7 +490,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -549,11 +549,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -577,7 +577,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -592,7 +592,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -651,10 +651,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -677,7 +677,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1036,10 +1036,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1062,7 +1062,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1150,11 +1150,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1179,7 +1179,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1194,7 +1194,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1253,11 +1253,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1281,7 +1281,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1296,7 +1296,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1456,11 +1456,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1485,7 +1485,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1500,7 +1500,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1559,11 +1559,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1587,7 +1587,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1606,7 +1606,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucket_many.q.out ql/src/test/results/clientpositive/llap/bucket_many.q.out index 72b0ec4..28b9424 100644 --- ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 964d058..5896005 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -125,10 +125,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -148,7 +148,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -168,7 +168,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -209,11 +209,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -233,26 +233,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -271,7 +252,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -282,7 +263,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -297,6 +278,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -346,11 +346,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 4 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -369,7 +369,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -389,7 +389,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -398,7 +398,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -411,7 +411,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d @@ -430,21 +445,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -498,11 +498,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 1 (CUSTOM_EDGE), Map 2 (CUSTOM_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 2 (CO_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d @@ -521,7 +521,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -540,7 +540,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -560,7 +560,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 0 Map 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -569,7 +569,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -582,7 +582,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -646,11 +646,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -676,7 +676,26 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -696,7 +715,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) @@ -711,25 +730,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) Stage: Stage-0 Fetch Operator @@ -756,11 +756,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -780,7 +780,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), substr(_col2, 5) (type: string) @@ -800,7 +800,41 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: tab @@ -820,7 +854,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: b @@ -840,40 +874,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 4 - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -900,11 +900,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -924,7 +924,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), substr(_col1, 5) (type: string) @@ -944,7 +944,41 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: y @@ -963,7 +997,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: b @@ -983,40 +1017,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 4 - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1041,10 +1041,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1066,8 +1066,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 1100 Data size: 20486 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1082,7 +1082,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1102,7 +1102,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1145,10 +1145,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1168,7 +1168,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1178,7 +1178,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1193,7 +1193,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: y @@ -1212,7 +1212,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1258,11 +1258,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -1288,7 +1288,22 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1308,7 +1323,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) @@ -1323,21 +1338,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -1364,11 +1364,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -1388,7 +1388,22 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1408,7 +1423,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) @@ -1423,21 +1438,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -1462,10 +1462,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1485,7 +1485,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1505,7 +1505,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1562,10 +1562,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1585,7 +1585,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1605,7 +1605,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1640,10 +1640,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1663,7 +1663,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1673,7 +1673,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -1688,7 +1688,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: c @@ -1707,7 +1707,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1751,10 +1751,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1786,10 +1786,10 @@ STAGE PLANS: Target Input: b Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Target Vertex: Map 2 + Target Vertex: Vertex 2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1809,7 +1809,7 @@ STAGE PLANS: 1 _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 60500 Data size: 12826000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string) @@ -1862,11 +1862,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1885,14 +1885,29 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count() + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1911,21 +1926,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 19 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count() - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1981,11 +1981,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2004,14 +2004,29 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count() + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2030,21 +2045,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 19 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count() - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out index f6500eb..ded7ccd 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out @@ -121,10 +121,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -144,7 +144,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -154,7 +154,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -169,7 +169,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: c @@ -188,7 +188,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -254,10 +254,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -277,7 +277,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -297,7 +297,7 @@ STAGE PLANS: 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -334,10 +334,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -357,7 +357,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 60 Data size: 1126 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -368,7 +368,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -409,10 +409,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -432,7 +432,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 182 Data size: 3400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -443,7 +443,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -484,10 +484,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -506,7 +506,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 1024 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -526,7 +526,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 182 Data size: 3400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -557,10 +557,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab @@ -580,26 +580,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -615,7 +596,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -624,6 +605,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -644,10 +644,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab @@ -667,27 +667,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -703,7 +683,7 @@ STAGE PLANS: 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string) @@ -716,6 +696,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out index e6cfb40..07df57d 100644 --- ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out +++ ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out @@ -22,7 +22,7 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@T2 -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM ( SELECT tmp1.name as name FROM ( SELECT name, 'MMM' AS n FROM T1) tmp1 diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out index 93efa3c..b0e12d7 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out @@ -41,10 +41,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -65,28 +65,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: key (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: value (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -125,6 +104,27 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: key (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: value (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -165,10 +165,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -189,28 +189,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: key (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: value (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -249,6 +228,27 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: key (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: value (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -383,10 +383,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,7 +458,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -532,55 +581,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -741,10 +741,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -816,7 +816,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -890,55 +939,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out index cea632b..74adfbd 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out @@ -111,10 +111,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -188,7 +188,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -262,55 +311,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -475,10 +475,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -552,7 +552,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -626,55 +675,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -856,10 +856,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -933,7 +933,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1056,55 +1105,6 @@ STAGE PLANS: Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] /srcbucket_mapjoin_part_2/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out index 4aa0e0e..fd6952a 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -212,7 +212,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -286,55 +335,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -499,10 +499,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -576,7 +576,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -650,55 +699,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out index 21ce1f8..f27803d 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -210,7 +210,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -282,55 +331,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -483,10 +483,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -558,7 +558,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -630,55 +679,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out index 3a2cbb2..8168a02 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out @@ -73,11 +73,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -151,7 +151,66 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_1 Truncated Path -> Alias: /srcbucket_mapjoin_part_1/ds=2008-04-08/hr=0 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col8 + Position of Big Table: 0 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: aa + sort order: ++ + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 1 + TopN Hash Memory Usage: 0.1 + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -226,65 +285,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08/hr=0 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col8 - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: aa - sort order: ++ - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 1 - TopN Hash Memory Usage: 0.1 - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucketpruning1.q.out ql/src/test/results/clientpositive/llap/bucketpruning1.q.out index 922f23a..040003e 100644 --- ql/src/test/results/clientpositive/llap/bucketpruning1.q.out +++ ql/src/test/results/clientpositive/llap/bucketpruning1.q.out @@ -29,7 +29,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -91,7 +91,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -153,7 +153,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -215,7 +215,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -277,7 +277,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -339,7 +339,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -401,7 +401,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -463,7 +463,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -525,7 +525,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -587,7 +587,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -649,7 +649,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -711,7 +711,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -773,7 +773,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -834,7 +834,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -895,7 +895,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -957,7 +957,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1018,7 +1018,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1079,7 +1079,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1140,7 +1140,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1201,7 +1201,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1262,7 +1262,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1323,7 +1323,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1384,7 +1384,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1444,7 +1444,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1505,7 +1505,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1566,7 +1566,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_unpruned @@ -1627,7 +1627,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_unpruned diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out index e4fed11..3690558 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out @@ -99,11 +99,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -123,7 +123,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -143,7 +143,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -157,7 +157,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -278,11 +278,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -302,7 +302,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -322,7 +322,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -336,7 +336,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -481,11 +481,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -505,7 +505,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -525,7 +525,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 184 Data size: 3097 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -539,7 +539,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -690,11 +690,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -714,7 +714,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -734,7 +734,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -748,7 +748,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -881,11 +881,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -905,7 +905,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -925,7 +925,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -939,7 +939,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1072,11 +1072,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -1096,7 +1096,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -1116,7 +1116,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) @@ -1130,7 +1130,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out index 67e925a..0bb8b77 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out @@ -77,11 +77,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -101,7 +101,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -121,7 +121,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -263,11 +263,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -287,7 +287,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -307,7 +307,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -321,7 +321,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -449,11 +449,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -473,7 +473,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -493,7 +493,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col6) (type: string) @@ -507,7 +507,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -570,11 +570,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -594,7 +594,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -614,7 +614,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col6) (type: string) @@ -628,7 +628,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -697,11 +697,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -721,7 +721,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -741,7 +741,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -755,7 +755,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -901,11 +901,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -925,7 +925,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -945,7 +945,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -959,7 +959,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1115,11 +1115,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1139,7 +1139,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1159,7 +1159,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -1173,7 +1173,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out index 8691d0d..f7f6e90 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out @@ -77,11 +77,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -101,7 +101,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -121,7 +121,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -260,11 +260,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -284,7 +284,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -304,7 +304,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -318,7 +318,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -449,11 +449,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -473,7 +473,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -493,7 +493,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 30 Data size: 515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -507,7 +507,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/cluster.q.out ql/src/test/results/clientpositive/llap/cluster.q.out index 8c89ee3..d0ee03a 100644 --- ql/src/test/results/clientpositive/llap/cluster.q.out +++ ql/src/test/results/clientpositive/llap/cluster.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -36,7 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -149,10 +149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -172,7 +172,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -217,10 +217,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -240,7 +240,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -285,10 +285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -308,7 +308,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -353,10 +353,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -376,7 +376,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,10 +421,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -444,7 +444,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -489,10 +489,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -512,7 +512,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -557,11 +557,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -577,22 +577,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -613,7 +598,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 4 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -627,6 +612,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -658,11 +658,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -678,23 +678,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -715,7 +699,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -729,6 +713,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -760,11 +760,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -780,23 +780,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -817,7 +801,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -831,6 +815,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -862,11 +862,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -882,22 +882,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -918,7 +903,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -932,6 +917,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -975,12 +975,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1000,7 +1002,21 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -1020,22 +1036,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out index 80ccddd..47df121 100644 --- ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out +++ ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -123,10 +123,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -308,10 +308,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -331,7 +331,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -508,10 +508,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -531,7 +531,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendarp @@ -717,7 +717,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -870,10 +870,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendarp @@ -893,7 +893,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/column_access_stats.q.out ql/src/test/results/clientpositive/llap/column_access_stats.q.out index c56c818..7b9bfd9 100644 --- ql/src/test/results/clientpositive/llap/column_access_stats.q.out +++ ql/src/test/results/clientpositive/llap/column_access_stats.q.out @@ -177,11 +177,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -199,7 +201,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1 @@ -217,8 +219,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -265,11 +265,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -287,7 +289,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1 @@ -305,8 +307,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -380,10 +380,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -402,7 +402,25 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -421,24 +439,6 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -511,10 +511,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -534,7 +534,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -554,24 +572,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -613,10 +613,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -635,7 +635,25 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -654,24 +672,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -726,11 +726,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -749,7 +749,41 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: t2 @@ -768,7 +802,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: t3 @@ -788,40 +822,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/constprog_dpp.q.out ql/src/test/results/clientpositive/llap/constprog_dpp.q.out index 8f51db3..23efdfb 100644 --- ql/src/test/results/clientpositive/llap/constprog_dpp.q.out +++ ql/src/test/results/clientpositive/llap/constprog_dpp.q.out @@ -41,35 +41,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 3 <- Union 4 (CONTAINS) -Map 6 <- Union 4 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 5 <- Union 4 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 4 (CONTAINS) +Vertex 5 <- Union 4 (PARTITION_EDGE) +Vertex 6 <- Union 4 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_17] Merge Join Operator [MERGEJOIN_21] (rows=1 width=0) Conds:RS_13._col0=RS_14._col0(Left Outer),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] PartitionCols:_col0 Select Operator [SEL_1] (rows=1 width=0) Output:["_col0"] TableScan [TS_0] (rows=1 width=0) default@tb1,a,Tbl:PARTIAL,Col:NONE,Output:["id"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col0 Limit [LIM_11] (rows=1 width=0) Number of rows:1 Select Operator [SEL_10] (rows=1 width=0) Output:["_col0"] - <-Union 4 [CUSTOM_SIMPLE_EDGE] - <-Map 3 [CONTAINS] llap + <-Union 4 [PARTITION_EDGE] + <-Vertex 3 [CONTAINS] llap Reduce Output Operator [RS_9] Limit [LIM_8] (rows=1 width=0) Number of rows:1 @@ -77,7 +77,7 @@ Stage-0 Output:["_col0"] TableScan [TS_2] (rows=1 width=0) Output:["id"] - <-Map 6 [CONTAINS] llap + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_9] Limit [LIM_8] (rows=1 width=0) Number of rows:1 diff --git ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out index 98d2328..e1a8e89 100644 --- ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out @@ -54,20 +54,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Select Operator [SEL_11] (rows=5 width=22) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8._col3=RS_9._col0(Left Semi),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col3 Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col2","_col3"] @@ -75,8 +75,8 @@ Stage-0 predicate:((val = 't1val01') and dimid is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Group By Operator [GBY_7] (rows=5 width=3) Output:["_col0"],keys:_col0 @@ -107,35 +107,26 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_18] Select Operator [SEL_17] (rows=5 width=24) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_28] (rows=5 width=24) Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col0","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=3 width=10) - predicate:id is not null - TableScan [TS_6] (rows=3 width=10) - default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_27] (rows=5 width=22) Conds:RS_11._col2=RS_12._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col2 Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col2"] @@ -143,8 +134,8 @@ Stage-0 predicate:((val = 't1val01') and id is not null and dimid is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","dimid"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_10] (rows=5 width=3) Output:["_col0"],keys:_col0 @@ -154,6 +145,15 @@ Stage-0 predicate:id is not null TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=3 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_26] (rows=3 width=10) + predicate:id is not null + TableScan [TS_6] (rows=3 width=10) + default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id PREHOOK: type: QUERY @@ -177,35 +177,26 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_18] Select Operator [SEL_17] (rows=5 width=24) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_28] (rows=5 width=24) Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col0","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=3 width=10) - predicate:id is not null - TableScan [TS_6] (rows=3 width=10) - default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_27] (rows=5 width=22) Conds:RS_11._col2=RS_12._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col2 Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col2"] @@ -213,8 +204,8 @@ Stage-0 predicate:((val = 't1val01') and dimid is not null and id is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","dimid"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_10] (rows=5 width=3) Output:["_col0"],keys:_col0 @@ -224,6 +215,15 @@ Stage-0 predicate:id is not null TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=3 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_26] (rows=3 width=10) + predicate:id is not null + TableScan [TS_6] (rows=3 width=10) + default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id PREHOOK: type: QUERY @@ -247,18 +247,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:100, true Select Operator [SEL_2] (rows=1 width=20) Output:["_col0","_col1","_col2"] @@ -266,8 +266,8 @@ Stage-0 predicate:false TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 @@ -295,18 +295,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:100, true Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col1","_col2"] @@ -314,8 +314,8 @@ Stage-0 predicate:((dimid = 100) and (dimid = 100) is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 @@ -345,18 +345,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:100, true Select Operator [SEL_2] (rows=1 width=20) Output:["_col0","_col1","_col2"] @@ -364,8 +364,8 @@ Stage-0 predicate:false TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 @@ -393,18 +393,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:100, true Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col1","_col2"] @@ -412,8 +412,8 @@ Stage-0 predicate:((dimid = 100) and (dimid = 100) is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 @@ -443,18 +443,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:100, true Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col1","_col2"] @@ -462,8 +462,8 @@ Stage-0 predicate:((dimid = 100) and (dimid = 100) is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out index 86d94b4..ebd7018 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out @@ -19,12 +19,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -43,26 +43,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +66,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -107,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -122,6 +103,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -167,12 +167,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -191,26 +191,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -233,7 +214,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -255,7 +236,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -270,6 +251,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -315,12 +315,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -339,7 +339,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: y @@ -359,7 +359,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -375,7 +375,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -397,7 +397,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -457,12 +457,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -481,31 +481,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -528,7 +504,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -550,7 +526,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -565,6 +541,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -610,12 +610,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -634,31 +634,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -681,7 +657,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -703,7 +679,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -718,6 +694,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -763,12 +763,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -784,23 +784,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -823,7 +807,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -845,7 +829,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -860,6 +844,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -905,12 +905,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -926,23 +926,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -965,7 +949,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -987,7 +971,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1002,6 +986,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1047,12 +1047,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1068,23 +1068,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1111,7 +1095,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1133,7 +1117,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1148,6 +1132,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1193,12 +1193,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1214,23 +1214,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1257,7 +1241,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1279,7 +1263,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1294,6 +1278,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1337,11 +1337,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1357,23 +1357,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1400,7 +1384,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 12 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1416,6 +1400,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1472,11 +1472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1492,23 +1492,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1535,7 +1519,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 12 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1551,6 +1535,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1609,12 +1609,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1630,23 +1630,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1673,7 +1657,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1695,7 +1679,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1710,6 +1694,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1755,12 +1755,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1776,23 +1776,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1819,7 +1803,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1841,7 +1825,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1856,6 +1840,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1901,12 +1901,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1922,23 +1922,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1961,7 +1945,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1983,7 +1967,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1998,6 +1982,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2043,12 +2043,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2064,23 +2064,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2103,7 +2087,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2125,7 +2109,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2140,6 +2124,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2185,12 +2185,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2206,23 +2206,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2245,7 +2229,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2267,7 +2251,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2282,6 +2266,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2327,12 +2327,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2348,23 +2348,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2387,7 +2371,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2409,7 +2393,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2424,6 +2408,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2469,12 +2469,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2494,26 +2494,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2536,7 +2517,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5490 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2558,7 +2539,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2573,6 +2554,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2618,12 +2618,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2643,26 +2643,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2685,7 +2666,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5490 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2707,7 +2688,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2722,6 +2703,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2767,12 +2767,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2791,26 +2791,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2833,7 +2814,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2855,7 +2836,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2870,6 +2851,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2915,12 +2915,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2939,26 +2939,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2981,7 +2962,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3003,7 +2984,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3018,6 +2999,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index b628cb1..41afbe8 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -21,13 +21,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -49,29 +49,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -86,7 +64,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -110,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -125,7 +103,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -189,13 +189,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -217,29 +217,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -254,7 +232,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -278,7 +256,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -293,7 +271,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -357,13 +357,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -386,26 +386,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -420,7 +401,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -444,7 +425,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -459,7 +440,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -523,13 +523,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -552,26 +552,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -586,7 +567,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -610,7 +591,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -625,7 +606,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -689,13 +689,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -714,30 +714,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -752,7 +729,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -776,7 +753,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -791,7 +768,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -855,13 +855,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -880,30 +880,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -918,7 +895,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -942,7 +919,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -957,7 +934,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1021,13 +1021,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1050,30 +1050,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1088,7 +1065,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1112,7 +1089,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1127,7 +1104,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1191,13 +1191,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1220,30 +1220,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1258,7 +1235,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1282,7 +1259,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1297,7 +1274,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1363,14 +1363,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1391,28 +1391,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1425,7 +1404,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1448,7 +1427,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1470,7 +1449,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1485,7 +1464,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1551,14 +1551,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1579,28 +1579,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1613,7 +1592,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1636,7 +1615,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1658,7 +1637,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1673,7 +1652,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1737,13 +1737,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1762,49 +1762,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1825,7 +1783,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1849,7 +1807,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1864,7 +1822,49 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1928,13 +1928,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1953,49 +1953,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2016,7 +1974,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2040,7 +1998,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2055,7 +2013,49 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out index d3cfce8..0fbbdbe 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out @@ -21,14 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -47,65 +47,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -126,7 +68,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -150,7 +92,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -165,7 +107,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -192,7 +173,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -207,6 +188,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -256,53 +256,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -321,26 +282,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -361,7 +303,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -385,7 +327,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -400,7 +342,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -427,7 +408,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -442,6 +423,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -491,13 +491,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -517,7 +517,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -531,7 +531,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 10980 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0) (type: int), hash(_col3) (type: int), hash(_col1) (type: int) @@ -548,7 +548,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -568,7 +583,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -588,7 +603,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -608,7 +623,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: x @@ -627,36 +657,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -691,68 +691,29 @@ FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value ON b.key = d.key) tmp PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)), SUM(HASH(tmp.value)) -FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value - FROM (SELECT x.key, x.value FROM src1 x JOIN src y ON (x.key = y.key)) b - JOIN (SELECT x.key, count(1) AS cnt FROM src1 x JOIN src y ON (x.key = y.key) group by x.key) d - ON b.key = d.key) tmp -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 +SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)), SUM(HASH(tmp.value)) +FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value + FROM (SELECT x.key, x.value FROM src1 x JOIN src y ON (x.key = y.key)) b + JOIN (SELECT x.key, count(1) AS cnt FROM src1 x JOIN src y ON (x.key = y.key) group by x.key) d + ON b.key = d.key) tmp +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: y @@ -771,26 +732,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -811,7 +753,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -835,7 +777,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -850,7 +792,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -877,7 +858,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -892,6 +873,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -941,53 +941,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1006,26 +967,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1046,7 +988,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1070,7 +1012,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1085,7 +1027,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1112,7 +1093,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1127,6 +1108,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1176,13 +1176,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1202,7 +1202,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -1216,7 +1216,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 10980 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col2) (type: int), hash(_col3) (type: int), hash(_col1) (type: int) @@ -1233,7 +1233,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -1253,7 +1268,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -1273,7 +1288,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -1293,7 +1308,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: x @@ -1312,36 +1342,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out index 2034f8e..e745752a 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out @@ -67,12 +67,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -91,45 +91,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -158,7 +120,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -180,7 +142,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -195,6 +157,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -242,12 +242,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -266,45 +266,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -333,7 +295,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -355,7 +317,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -370,6 +332,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -417,12 +417,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -444,8 +444,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 4 - 2 Map 5 + 1 Vertex 4 + 2 Vertex 5 Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) @@ -465,7 +465,44 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hash(_col0) (type: int), hash(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col0), sum(_col1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y @@ -484,7 +521,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z @@ -503,43 +540,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hash(_col0) (type: int), hash(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(_col0), sum(_col1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -587,12 +587,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -608,39 +608,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -665,7 +633,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -687,7 +655,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -702,6 +670,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -749,12 +749,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -770,39 +770,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -827,7 +795,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -849,7 +817,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -864,6 +832,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -911,12 +911,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -932,39 +932,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -993,7 +961,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1015,7 +983,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1030,6 +998,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1077,12 +1077,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1098,39 +1098,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1159,7 +1127,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1181,21 +1149,53 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1243,12 +1243,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1264,39 +1264,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1325,7 +1293,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1347,7 +1315,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1362,6 +1330,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1409,12 +1409,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1430,39 +1430,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1491,7 +1459,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1513,7 +1481,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1528,6 +1496,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1575,12 +1575,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1596,39 +1596,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1657,7 +1625,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1679,7 +1647,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1694,6 +1662,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1741,12 +1741,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1762,39 +1762,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1823,7 +1791,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1845,7 +1813,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1860,6 +1828,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out index 4fec286..45e7e50 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out @@ -23,14 +23,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -49,64 +49,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -129,7 +72,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -144,7 +87,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -162,7 +105,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -185,7 +166,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -200,6 +181,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -267,14 +267,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -293,64 +293,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -373,7 +316,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -388,7 +331,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -406,7 +349,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -429,7 +410,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -444,6 +425,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -511,13 +511,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 2 (BROADCAST_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -537,7 +537,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 44 Data size: 3784 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -553,7 +553,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 Map Operator Tree: TableScan alias: y @@ -572,7 +587,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -592,7 +607,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -608,7 +623,33 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 0 Vertex 2 + Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -627,47 +668,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -733,11 +733,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -756,7 +756,25 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -782,25 +800,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -878,11 +878,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -901,7 +901,25 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -927,25 +945,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1023,12 +1023,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -1047,29 +1047,28 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: y + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -1085,25 +1084,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1126,7 +1107,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1141,6 +1122,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1204,12 +1204,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -1228,29 +1228,28 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: y + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -1266,25 +1265,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1307,7 +1288,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1322,6 +1303,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1385,12 +1385,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1409,45 +1409,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: yy - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1470,7 +1432,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1485,7 +1447,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1503,6 +1465,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: yy + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1588,12 +1588,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1612,45 +1612,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: yy - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1673,7 +1635,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1688,7 +1650,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1706,6 +1668,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: yy + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1793,12 +1793,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: zz @@ -1817,7 +1817,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: xx @@ -1836,7 +1860,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -1855,50 +1879,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1921,7 +1902,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1936,6 +1917,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2025,12 +2025,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: zz @@ -2049,7 +2049,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: xx @@ -2068,7 +2092,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2087,50 +2111,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2153,7 +2134,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2168,6 +2149,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2257,12 +2257,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2281,64 +2281,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: zz - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: xx - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2361,7 +2304,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2376,7 +2319,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2400,6 +2343,63 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: zz + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: xx + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2461,81 +2461,43 @@ POSTHOOK: Input: default@src1 406 406 16 406 406 16 406 406 16 -66 66 1 -98 98 4 -98 98 4 -PREHOOK: query: EXPLAIN -SELECT xx.key, yy.key, yy.cnt -FROM src1 xx -JOIN -(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy -ON xx.key=yy.key JOIN src zz -ON yy.key=zz.key -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT xx.key, yy.key, yy.cnt -FROM src1 xx -JOIN -(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy -ON xx.key=yy.key JOIN src zz -ON yy.key=zz.key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 +66 66 1 +98 98 4 +98 98 4 +PREHOOK: query: EXPLAIN +SELECT xx.key, yy.key, yy.cnt +FROM src1 xx +JOIN +(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy +ON xx.key=yy.key JOIN src zz +ON yy.key=zz.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT xx.key, yy.key, yy.cnt +FROM src1 xx +JOIN +(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy +ON xx.key=yy.key JOIN src zz +ON yy.key=zz.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: zz + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -2551,26 +2513,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: xx - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2593,7 +2536,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2608,7 +2551,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2632,6 +2575,63 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: zz + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: xx + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2723,14 +2723,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2752,49 +2752,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2809,7 +2767,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2832,7 +2790,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2847,7 +2805,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2869,7 +2827,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2884,6 +2864,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 8 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2977,14 +2977,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3006,49 +3006,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3063,7 +3021,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3086,7 +3044,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3101,7 +3059,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3123,7 +3081,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3138,6 +3118,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 8 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3229,15 +3229,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3256,7 +3256,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -3275,46 +3275,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3337,7 +3298,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3352,7 +3313,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3367,7 +3328,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) sort order: +++++ Statistics: Num rows: 41 Data size: 11480 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3381,7 +3342,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3404,7 +3404,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3486,14 +3486,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3512,65 +3512,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3593,7 +3535,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3608,7 +3550,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3626,7 +3568,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3649,7 +3630,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3664,6 +3645,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3731,13 +3731,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 2 (BROADCAST_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3757,7 +3757,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 44 Data size: 3784 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -3773,7 +3773,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 Map Operator Tree: TableScan alias: y @@ -3792,7 +3807,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -3812,7 +3827,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -3828,41 +3843,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3879,7 +3860,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 41 Data size: 11480 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -3888,6 +3869,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/count.q.out ql/src/test/results/clientpositive/llap/count.q.out index 2953718..2e88b2b 100644 --- ql/src/test/results/clientpositive/llap/count.q.out +++ ql/src/test/results/clientpositive/llap/count.q.out @@ -42,10 +42,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -68,7 +68,7 @@ STAGE PLANS: value expressions: _col5 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -185,10 +185,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -271,7 +271,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -315,10 +315,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -340,7 +340,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -384,11 +384,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -404,7 +404,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -418,7 +418,7 @@ STAGE PLANS: Map-reduce partition columns: a (type: int) Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: b (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -470,11 +470,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -490,7 +490,7 @@ STAGE PLANS: Statistics: Num rows: 19 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -503,7 +503,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: b (type: int) Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -555,11 +555,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -575,7 +575,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -589,7 +589,7 @@ STAGE PLANS: Map-reduce partition columns: c (type: int) Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: b (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -644,10 +644,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -664,7 +664,7 @@ STAGE PLANS: value expressions: b (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -719,10 +719,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -739,7 +739,7 @@ STAGE PLANS: value expressions: $f3 (type: int), $f4 (type: int), $f5 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,10 +787,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -807,7 +807,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -855,10 +855,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -873,7 +873,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/cross_join.q.out ql/src/test/results/clientpositive/llap/cross_join.q.out index ae3f9bf..3a0521e 100644 --- ql/src/test/results/clientpositive/llap/cross_join.q.out +++ ql/src/test/results/clientpositive/llap/cross_join.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -12,10 +12,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -30,19 +30,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -60,6 +48,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -67,7 +67,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,19 +99,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -129,6 +117,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -149,10 +149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -171,7 +171,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -190,24 +208,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -215,7 +215,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -229,10 +229,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -249,7 +249,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 250000 Data size: 21750000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -260,7 +260,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -279,7 +279,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 @@ -293,10 +293,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -313,7 +313,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 250000 Data size: 21750000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -324,7 +324,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -356,10 +356,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -379,7 +379,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -390,7 +390,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 diff --git ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out index 2bc17cb..e9e92b2 100644 --- ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out @@ -28,7 +28,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -42,10 +42,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -60,22 +60,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -93,6 +78,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -100,7 +100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -114,11 +114,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d1 @@ -138,7 +138,40 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d2 @@ -158,7 +191,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -173,39 +206,6 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -213,7 +213,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -233,12 +233,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -253,29 +253,28 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: d2 + alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) @@ -291,25 +290,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -330,7 +311,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -342,6 +323,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 6 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -349,8 +349,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -364,12 +364,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -384,34 +384,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -429,7 +402,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -450,7 +438,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 100 Data size: 1540 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -462,6 +450,18 @@ STAGE PLANS: sort order: Statistics: Num rows: 50 Data size: 770 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 6 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -469,7 +469,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -487,13 +487,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -514,45 +514,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -564,7 +526,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -582,7 +544,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -603,7 +584,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -615,6 +596,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out index c865788..8308f82 100644 --- ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out @@ -28,7 +28,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -42,10 +42,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -62,7 +62,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 5000 Data size: 106120 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -73,7 +73,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -95,7 +95,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -109,11 +109,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d1 @@ -133,7 +133,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d2 @@ -161,7 +161,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: a @@ -178,7 +178,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 5500 Data size: 116432 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -196,7 +196,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -216,12 +216,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -238,7 +238,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Reducer 3 + 1 Vertex 3 Statistics: Num rows: 2500 Data size: 52560 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -249,7 +249,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -269,7 +269,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -283,7 +283,19 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Vertex 4 Map Operator Tree: TableScan alias: d2 @@ -302,18 +314,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -321,8 +321,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Vertex 2' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -336,12 +336,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -358,7 +358,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Reducer 3 + 1 Vertex 3 Statistics: Num rows: 25000 Data size: 655600 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -369,7 +369,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -386,7 +386,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 100 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -400,19 +400,7 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -424,6 +412,18 @@ STAGE PLANS: sort order: Statistics: Num rows: 50 Data size: 730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 4 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -431,7 +431,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -449,12 +449,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -475,7 +475,32 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: d1 @@ -495,7 +520,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -509,7 +534,19 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Vertex 5 Map Operator Tree: TableScan alias: d2 @@ -528,43 +565,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/ctas.q.out ql/src/test/results/clientpositive/llap/ctas.q.out index fe492e4..379fd86 100644 --- ql/src/test/results/clientpositive/llap/ctas.q.out +++ ql/src/test/results/clientpositive/llap/ctas.q.out @@ -30,11 +30,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -50,7 +50,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -65,7 +65,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -186,11 +186,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -206,7 +206,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -221,7 +221,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -342,11 +342,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -362,7 +362,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -377,7 +377,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -562,11 +562,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -582,7 +582,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -597,7 +597,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -720,11 +720,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -740,7 +740,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -755,7 +755,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/cte_3.q.out ql/src/test/results/clientpositive/llap/cte_3.q.out index b6cda54..6a68794 100644 --- ql/src/test/results/clientpositive/llap/cte_3.q.out +++ ql/src/test/results/clientpositive/llap/cte_3.q.out @@ -13,7 +13,7 @@ Plan optimized by CBO. Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.q1"} Select Operator [SEL_2] (rows=2 width=85) @@ -67,7 +67,7 @@ Plan optimized by CBO. Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.q1"} Select Operator [SEL_2] (rows=2 width=85) @@ -121,7 +121,7 @@ Plan optimized by CBO. Stage-5 Dependency Collection{} Stage-4 - Map 2 llap + Vertex 2 llap File Output Operator [FS_8] table:{"name:":"default.q1"} Select Operator [SEL_7] (rows=1 width=0) @@ -133,7 +133,7 @@ Stage-5 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.q2"} Select Operator [SEL_2] (rows=2 width=85) diff --git ql/src/test/results/clientpositive/llap/cte_5.q.out ql/src/test/results/clientpositive/llap/cte_5.q.out index 58a963b..428d6e0 100644 --- ql/src/test/results/clientpositive/llap/cte_5.q.out +++ ql/src/test/results/clientpositive/llap/cte_5.q.out @@ -61,7 +61,7 @@ src_thrift srcbucket srcbucket2 srcpart -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.colnum @@ -77,34 +77,34 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_10] Select Operator [SEL_9] (rows=2 width=89) Output:["_col0"] Merge Join Operator [MERGEJOIN_13] (rows=2 width=89) Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_6] Select Operator [SEL_2] (rows=1 width=3) Filter Operator [FIL_11] (rows=1 width=3) predicate:(UDFToDouble(colnum) = 5.0) TableScan [TS_0] (rows=1 width=3) mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_7] Select Operator [SEL_5] (rows=2 width=85) Filter Operator [FIL_12] (rows=2 width=87) predicate:(key = '5') TableScan [TS_3] (rows=500 width=87) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: with q1 as (select * from src where key= '5') select a.colnum from mydb.q1 as a join q1 as b diff --git ql/src/test/results/clientpositive/llap/cte_mat_1.q.out ql/src/test/results/clientpositive/llap/cte_mat_1.q.out index a1156e8..760690a 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_1.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_1.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -14,30 +14,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=4 width=85) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) - Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=2 width=85) - Filter Operator [FIL_11] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/cte_mat_2.q.out ql/src/test/results/clientpositive/llap/cte_mat_2.q.out index a1156e8..7099592 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_2.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_2.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -14,27 +14,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_10] Select Operator [SEL_9] (rows=4 width=85) Output:["_col0"] Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_6] Select Operator [SEL_2] (rows=2 width=85) Filter Operator [FIL_11] (rows=2 width=87) predicate:(key = '5') TableScan [TS_0] (rows=500 width=87) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_7] Select Operator [SEL_5] (rows=2 width=85) Filter Operator [FIL_12] (rows=2 width=87) predicate:(key = '5') diff --git ql/src/test/results/clientpositive/llap/cte_mat_3.q.out ql/src/test/results/clientpositive/llap/cte_mat_3.q.out index cb88f0c..737c206 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_3.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_3.q.out @@ -13,18 +13,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_15] Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Select Operator [SEL_7] (rows=1 width=0) Output:["_col0"] @@ -32,8 +32,8 @@ Stage-3 predicate:key is not null TableScan [TS_5] (rows=1 width=0) default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Select Operator [SEL_10] (rows=1 width=0) Output:["_col0"] @@ -44,7 +44,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.q1"} Select Operator [SEL_2] (rows=2 width=176) diff --git ql/src/test/results/clientpositive/llap/cte_mat_4.q.out ql/src/test/results/clientpositive/llap/cte_mat_4.q.out index 1c5be08..fa319f7 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_4.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_4.q.out @@ -52,18 +52,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_15] Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Select Operator [SEL_7] (rows=1 width=0) Output:["_col0"] @@ -71,8 +71,8 @@ Stage-3 predicate:key is not null TableScan [TS_5] (rows=1 width=0) default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Select Operator [SEL_10] (rows=1 width=0) Output:["_col0"] @@ -83,7 +83,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.q1"} Select Operator [SEL_2] (rows=2 width=176) @@ -202,18 +202,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_15] Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Select Operator [SEL_7] (rows=1 width=0) Output:["_col0"] @@ -221,8 +221,8 @@ Stage-3 predicate:key is not null TableScan [TS_5] (rows=1 width=0) default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Select Operator [SEL_10] (rows=1 width=0) Output:["_col0"] @@ -233,7 +233,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.q1"} Select Operator [SEL_2] (rows=2 width=176) diff --git ql/src/test/results/clientpositive/llap/cte_mat_5.q.out ql/src/test/results/clientpositive/llap/cte_mat_5.q.out index 0c22b72..2d63e89 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_5.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_5.q.out @@ -76,46 +76,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=3) - Conds:RS_11.UDFToDouble(_col0)=RS_12.UDFToDouble(_col0)(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_7] (rows=1 width=3) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=3) - predicate:colnum is not null - TableScan [TS_5] (rows=1 width=3) - mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index ba14835..e754425 100644 --- ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -105,20 +105,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_10] Select Operator [SEL_9] (rows=5 width=228) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_15] (rows=5 width=228) Conds:RS_6._col1=RS_7._col3(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col1 Select Operator [SEL_2] (rows=5 width=88) Output:["_col0","_col1"] @@ -126,8 +126,8 @@ Stage-0 predicate:item is not null TableScan [TS_0] (rows=5 width=88) default@testdeci2,s,Tbl:COMPLETE,Col:COMPLETE,Output:["id","item"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col3 Select Operator [SEL_5] (rows=1 width=312) Output:["_col1","_col2","_col3"] diff --git ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index ca03652..9de6ff4 100644 --- ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -204,10 +204,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -226,7 +226,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 1b6bb1f..f059f7c 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -50,10 +50,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -74,7 +74,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -211,11 +211,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -232,7 +232,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -264,43 +298,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -340,11 +340,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -361,27 +361,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -400,7 +380,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -415,6 +395,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -467,11 +467,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -488,7 +488,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -520,43 +554,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: day(ds) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 day(_col0) (type: int) - 1 day(_col0) (type: int) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -596,11 +596,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -617,27 +617,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: day(_col0) (type: int) - sort order: + - Map-reduce partition columns: day(_col0) (type: int) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -656,7 +636,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -671,6 +651,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -712,12 +712,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -734,7 +734,57 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: srcpart_date @@ -766,10 +816,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: srcpart_hour @@ -801,59 +851,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -899,12 +899,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -922,47 +922,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -978,7 +938,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -997,7 +957,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1012,6 +972,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1066,11 +1066,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1086,7 +1086,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -1118,7 +1152,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -1133,43 +1167,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1209,11 +1209,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1230,27 +1230,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1269,7 +1249,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1284,6 +1264,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date_hour + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1334,11 +1334,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1355,7 +1355,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -1387,43 +1421,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1463,11 +1463,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1484,27 +1484,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1523,7 +1503,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1538,6 +1518,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1586,11 +1586,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1607,7 +1607,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -1639,43 +1673,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1715,11 +1715,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1736,7 +1736,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (UDFToDouble(_col0) * 2.0) (type: double) + 1 _col0 (type: double) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -1768,43 +1802,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (UDFToDouble(_col0) * 2.0) (type: double) - 1 _col0 (type: double) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1844,11 +1844,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1865,27 +1865,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_double_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1904,7 +1884,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1919,6 +1899,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_double_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1958,11 +1958,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1979,27 +1979,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_double_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2018,7 +1998,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2033,11 +2013,31 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_double_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: ListSink PREHOOK: query: select count(*) from srcpart join srcpart_double_hour on (srcpart.hr*2 = srcpart_double_hour.hr) where srcpart_double_hour.hour = 11 @@ -2085,11 +2085,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2106,7 +2106,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) + 1 UDFToString(_col0) (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -2138,43 +2172,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) - 1 UDFToString(_col0) (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2214,7 +2214,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' @@ -2228,12 +2228,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2246,27 +2246,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2285,7 +2265,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2300,7 +2280,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2320,7 +2320,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2347,7 +2347,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) @@ -2361,11 +2361,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2380,26 +2380,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) - Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2424,7 +2405,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2439,6 +2420,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date_hour + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) + Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2446,7 +2446,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2479,11 +2479,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2499,7 +2499,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -2531,7 +2565,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -2546,43 +2580,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2622,11 +2622,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2643,7 +2643,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -2675,43 +2709,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2732,11 +2732,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -2768,26 +2768,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2806,7 +2790,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2821,6 +2805,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2841,11 +2841,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2861,7 +2861,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -2893,43 +2927,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2952,12 +2952,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2974,7 +2974,57 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: srcpart_date @@ -3006,10 +3056,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: srcpart_hour @@ -3019,81 +3069,31 @@ STAGE PLANS: predicate: ((UDFToDouble(hour) = 11.0) and (UDFToDouble(hr) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart - Partition key expr: hr - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart + Partition key expr: hr + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Vertex 1 Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3135,14 +3135,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 2 (PARTITION_EDGE) + Vertex 7 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3163,47 +3163,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart_hour - filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3232,7 +3192,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3251,7 +3211,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3266,7 +3226,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3278,6 +3238,46 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart_hour + filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3313,14 +3313,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3337,47 +3339,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3396,7 +3358,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3411,7 +3373,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3432,7 +3414,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3459,8 +3441,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 - Reducer 9 + Target Vertex: Vertex 1 + Vertex 8 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3481,8 +3483,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -3520,14 +3520,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3544,47 +3546,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3605,7 +3567,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3620,7 +3582,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3641,7 +3623,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3668,8 +3650,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 - Reducer 9 + Target Vertex: Vertex 1 + Vertex 8 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3690,8 +3692,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -3730,53 +3730,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE), Union 9 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 9 (CONTAINS) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 9 (CONTAINS), Vertex 11 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 9 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3794,27 +3760,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3841,7 +3787,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -3856,8 +3802,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 5 - Reducer 12 + Target Vertex: Vertex 5 + Vertex 11 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3878,7 +3844,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3891,7 +3857,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3909,7 +3875,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ds (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3922,7 +3906,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3943,10 +3947,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -3987,11 +3987,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4008,7 +4008,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4021,7 +4021,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4053,24 +4068,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4123,11 +4123,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4144,7 +4144,7 @@ STAGE PLANS: 0 day(_col0) (type: int) 1 day(_col0) (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4157,7 +4157,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4189,24 +4204,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: day(ds) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4248,11 +4248,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4269,7 +4269,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -4278,7 +4278,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4291,7 +4291,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4323,10 +4338,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4358,24 +4373,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4430,11 +4430,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4450,7 +4450,7 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col2 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4463,7 +4463,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -4495,7 +4510,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -4510,24 +4525,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4578,11 +4578,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4599,7 +4599,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4612,7 +4612,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4644,24 +4659,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4701,11 +4701,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4722,7 +4722,7 @@ STAGE PLANS: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4735,7 +4735,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -4767,24 +4782,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4824,11 +4824,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4845,7 +4845,7 @@ STAGE PLANS: 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4858,7 +4858,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -4890,24 +4905,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4947,7 +4947,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' @@ -4961,11 +4961,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4978,7 +4978,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart @@ -4998,7 +4998,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5015,7 +5015,7 @@ STAGE PLANS: 0 1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 500000 Data size: 11124000 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5026,7 +5026,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5048,7 +5048,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -5088,11 +5088,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5109,7 +5109,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5122,7 +5122,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -5154,24 +5169,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5192,11 +5192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -5216,7 +5216,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5229,7 +5229,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -5245,21 +5260,6 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5280,11 +5280,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5300,7 +5300,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart_date @@ -5320,7 +5320,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5333,7 +5333,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5370,11 +5370,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5391,7 +5391,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -5400,7 +5400,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5413,7 +5413,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -5445,10 +5460,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -5480,24 +5495,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5539,11 +5539,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5564,7 +5564,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart_date @@ -5585,7 +5585,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -5594,7 +5594,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1 Data size: 444 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5607,7 +5607,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -5627,21 +5642,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5677,13 +5677,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Map 1 (BROADCAST_EDGE), Union 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Vertex 1 (BROADCAST_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5700,7 +5702,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart @@ -5720,27 +5722,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5761,7 +5743,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5777,7 +5759,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -5789,7 +5771,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5804,7 +5786,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5825,8 +5827,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -5900,11 +5900,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_orc @@ -5920,7 +5920,7 @@ STAGE PLANS: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) 1 _col0 (type: string), UDFToDouble(_col2) (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 413600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5933,7 +5933,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -5965,7 +5980,7 @@ STAGE PLANS: Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: UDFToDouble(_col2) (type: double) outputColumnNames: _col0 @@ -5980,24 +5995,9 @@ STAGE PLANS: Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index 9dab93d..0c05a8f 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -156,12 +156,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -179,7 +179,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -199,7 +199,35 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d1 @@ -232,37 +260,9 @@ STAGE PLANS: Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -329,12 +329,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -352,7 +352,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -372,28 +372,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d1 - filterExpr: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id (type: int), label (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -407,7 +386,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,6 +400,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: d1 + filterExpr: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: int), label (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -479,10 +479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -500,7 +500,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) @@ -515,7 +515,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -574,7 +574,7 @@ bar baz baz baz -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -617,7 +617,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 51 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -628,7 +628,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -651,7 +651,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -702,12 +702,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -725,7 +725,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -745,7 +745,35 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d1 @@ -778,37 +806,9 @@ STAGE PLANS: Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -867,11 +867,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: agg_01 @@ -889,7 +891,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -900,7 +902,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: dim_shops @@ -932,10 +934,10 @@ STAGE PLANS: Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: agg_01 @@ -953,7 +955,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -964,7 +966,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: dim_shops @@ -996,11 +998,9 @@ STAGE PLANS: Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1055,11 +1055,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1076,7 +1076,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2000000 Data size: 16000000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1089,7 +1089,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -1106,21 +1121,6 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out index cacde93..f91e9e6 100644 --- ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out @@ -115,11 +115,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -139,27 +139,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -178,7 +158,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -193,6 +173,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -232,13 +232,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -258,40 +258,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -310,7 +277,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,7 +292,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: all inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -376,11 +376,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -412,27 +412,10 @@ STAGE PLANS: Target Input: srcpart_small Partition key expr: ds Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 - Execution mode: llap - LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -451,7 +434,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -466,6 +449,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -505,11 +505,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -529,7 +529,43 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Statistics: Num rows: 27033 Data size: 2038454 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_small @@ -549,7 +585,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: alltypesorc_int @@ -569,42 +605,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - Statistics: Num rows: 27033 Data size: 2038454 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -646,16 +646,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 6 (BROADCAST_EDGE) - Map 8 <- Reducer 4 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 5 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 4 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -688,73 +688,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: alltypesorc_int - filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -775,7 +709,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -790,7 +724,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -802,7 +736,53 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: all inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -814,7 +794,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -826,6 +806,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 8 + Map Operator Tree: + TableScan + alias: alltypesorc_int + filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -867,11 +867,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -891,27 +891,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: (key1 is not null and value1 is not null) (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key1 is not null and value1 is not null) (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string), value1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -930,7 +910,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -945,6 +925,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: (key1 is not null and value1 is not null) (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key1 is not null and value1 is not null) (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string), value1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1003,14 +1003,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE), Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1030,7 +1030,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_small @@ -1066,51 +1100,17 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + LLAP IO: all inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1122,7 +1122,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1154,12 +1154,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1180,47 +1180,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: alltypesorc_int - filterExpr: cstring is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: cstring is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1236,7 +1196,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1255,7 +1215,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1270,6 +1230,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: alltypesorc_int + filterExpr: cstring is not null (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cstring is not null (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1311,16 +1311,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 7 (BROADCAST_EDGE) - Map 8 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 7 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 2 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 8 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1341,60 +1341,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: alltypesorc_int - filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1423,7 +1370,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1442,7 +1389,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1457,7 +1404,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1469,7 +1416,40 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: all inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1481,6 +1461,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 8 + Map Operator Tree: + TableScan + alias: alltypesorc_int + filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1522,12 +1522,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1547,7 +1547,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1560,7 +1560,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_small @@ -1593,22 +1608,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1659,12 +1659,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1684,7 +1684,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1697,7 +1697,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_small @@ -1730,22 +1745,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index eaa394d..61b5460 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -166,10 +166,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -189,7 +189,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -239,11 +239,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -265,7 +265,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -281,7 +281,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -331,10 +331,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -354,7 +354,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -403,10 +403,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -426,7 +426,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -543,10 +543,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -566,7 +566,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -616,11 +616,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -642,7 +642,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -658,7 +658,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -708,10 +708,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -731,7 +731,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -780,10 +780,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -803,7 +803,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1322,10 +1322,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1344,7 +1344,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1394,10 +1394,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1417,7 +1417,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1467,11 +1467,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1488,7 +1488,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1511,7 +1511,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1561,10 +1561,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1584,7 +1584,7 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1639,10 +1639,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1662,7 +1662,7 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2029,10 +2029,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -2052,7 +2052,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2101,10 +2101,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -2124,7 +2124,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2320,10 +2320,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc @@ -2341,7 +2341,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2538,10 +2538,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc @@ -2559,7 +2559,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index 95a4e0f..9488338 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -123,10 +123,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -196,11 +196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -222,7 +222,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -238,7 +238,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -311,7 +311,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -360,10 +360,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -383,7 +383,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -500,10 +500,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -523,7 +523,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -573,11 +573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -599,7 +599,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -615,7 +615,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -665,10 +665,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -688,7 +688,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -737,10 +737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -760,7 +760,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1279,10 +1279,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1301,7 +1301,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1351,10 +1351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1374,7 +1374,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1424,11 +1424,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1445,7 +1445,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1468,7 +1468,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1518,10 +1518,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1541,7 +1541,7 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1596,10 +1596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1619,7 +1619,7 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1986,10 +1986,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2009,7 +2009,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2058,10 +2058,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2081,7 +2081,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2432,10 +2432,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2455,7 +2455,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2506,10 +2506,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2529,7 +2529,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2580,10 +2580,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2603,7 +2603,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2654,10 +2654,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2677,7 +2677,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2728,10 +2728,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2751,7 +2751,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2802,10 +2802,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2825,7 +2825,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2876,7 +2876,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index 41a7709..08e0352 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -83,10 +83,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -106,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,10 +325,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -348,7 +348,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -562,10 +562,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -585,7 +585,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -804,10 +804,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -826,7 +826,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1095,10 +1095,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss_orc @@ -1118,7 +1118,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1337,10 +1337,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss_orc @@ -1359,7 +1359,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1605,10 +1605,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1631,7 +1631,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1735,10 +1735,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1761,7 +1761,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index b7679f1..a5c3055 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -107,7 +107,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -177,10 +177,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -200,7 +200,7 @@ STAGE PLANS: value expressions: _col3 (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -367,10 +367,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -389,7 +389,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -459,10 +459,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -481,7 +481,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 347200 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -657,10 +657,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -679,7 +679,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -750,10 +750,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -773,7 +773,7 @@ STAGE PLANS: value expressions: _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -873,10 +873,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -896,7 +896,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1070,10 +1070,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1092,7 +1092,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1163,10 +1163,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1185,7 +1185,7 @@ STAGE PLANS: Statistics: Num rows: 1600 Data size: 556800 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1285,10 +1285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1307,7 +1307,7 @@ STAGE PLANS: Statistics: Num rows: 1600 Data size: 422400 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1481,10 +1481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1504,7 +1504,7 @@ STAGE PLANS: value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1575,10 +1575,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1598,7 +1598,7 @@ STAGE PLANS: value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/empty_join.q.out ql/src/test/results/clientpositive/llap/empty_join.q.out index 19aa89f..d927a1f 100644 --- ql/src/test/results/clientpositive/llap/empty_join.q.out +++ ql/src/test/results/clientpositive/llap/empty_join.q.out @@ -55,40 +55,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Map Join Operator [MAPJOIN_22] (rows=2 width=1) - Conds:SEL_2._col0=RS_10._col0(Left Outer),SEL_2._col0=RS_11._col0(Inner),Output:["_col0","_col1","_col2"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1 width=0) - predicate:id is not null - TableScan [TS_3] (rows=1 width=0) - default@test_2,t2,Tbl:PARTIAL,Col:NONE,Output:["id"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1 width=1) - Output:["_col0"] - Filter Operator [FIL_21] (rows=1 width=1) - predicate:id is not null - TableScan [TS_6] (rows=1 width=1) - default@test_3,t3,Tbl:COMPLETE,Col:NONE,Output:["id"] - <-Select Operator [SEL_2] (rows=1 width=1) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=1) - predicate:id is not null - TableScan [TS_0] (rows=1 width=1) - default@test_1,t1,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 1 PREHOOK: query: SELECT t1.id, t2.id, t3.id FROM test_1 t1 diff --git ql/src/test/results/clientpositive/llap/except_distinct.q.out ql/src/test/results/clientpositive/llap/except_distinct.q.out index 94125b5..10136f8 100644 --- ql/src/test/results/clientpositive/llap/except_distinct.q.out +++ ql/src/test/results/clientpositive/llap/except_distinct.q.out @@ -206,12 +206,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -234,30 +236,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -286,7 +265,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -309,7 +288,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -338,8 +340,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -368,16 +368,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 14 <- Map 13 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Union 7 (CONTAINS), Vertex 13 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) + Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -400,53 +406,36 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count(1) + aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Vertex 11 Map Operator Tree: TableScan alias: src @@ -469,7 +458,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -485,49 +474,43 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: sum(_col2), sum(_col3) + aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 14 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -556,7 +539,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -585,7 +568,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -627,7 +610,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -669,7 +652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -692,12 +675,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -726,12 +726,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -752,28 +754,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -807,7 +788,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -830,7 +811,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -864,8 +866,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index c3f8071..0b716a3 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -53,7 +53,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_3] table:{"name:":"default.src_orc_merge_test_part"} Select Operator [SEL_1] (rows=500 width=95) @@ -78,7 +78,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -88,7 +88,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_7] table:{"name:":"default.src_orc_merge_test_part"} Select Operator [SEL_6] (rows=100 width=95) @@ -97,8 +97,8 @@ Stage-3 Number of rows:100 Select Operator [SEL_4] (rows=100 width=178) Output:["_col0","_col1"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Limit [LIM_2] (rows=100 width=178) Number of rows:100 Select Operator [SEL_1] (rows=500 width=178) @@ -123,18 +123,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Group By Operator [GBY_6] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_5] Group By Operator [GBY_4] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] Select Operator [SEL_2] (rows=500 width=102) @@ -157,18 +157,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Group By Operator [GBY_6] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_5] Group By Operator [GBY_4] (rows=1 width=8) Output:["_col0"],aggregations:["count(1)"] Select Operator [SEL_2] (rows=500 width=102) @@ -182,18 +182,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Group By Operator [GBY_6] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_5] Group By Operator [GBY_4] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] Select Operator [SEL_2] (rows=500 width=102) @@ -209,7 +209,7 @@ POSTHOOK: query: drop table src_orc_merge_test_part POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@src_orc_merge_test_part POSTHOOK: Output: default@src_orc_merge_test_part -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( select src1.key as k1, src1.value as v1, @@ -233,29 +233,29 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_17] Group By Operator [GBY_15] (rows=1 width=8) Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_14] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_14] Group By Operator [GBY_13] (rows=1 width=8) Output:["_col0"],aggregations:["sum(hash(_col0,_col1,_col2,_col3))"] Select Operator [SEL_11] (rows=27556 width=356) Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] Map Join Operator [MAPJOIN_20] (rows=27556 width=356) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_7] Select Operator [SEL_5] (rows=166 width=178) Output:["_col0","_col1"] @@ -277,20 +277,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_7] Select Operator [SEL_5] (rows=10 width=97) Output:["_col0","_col1","_col2"] Group By Operator [GBY_4] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_2] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -306,21 +306,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_12] Select Operator [SEL_11] (rows=5 width=20) Output:["_col0","_col1","_col2"] Group By Operator [GBY_10] (rows=5 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_8] (rows=5 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 @@ -328,8 +328,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_4] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_2] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -345,31 +345,31 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap + Vertex 7 llap File Output Operator [FS_42] Select Operator [SEL_41] (rows=1 width=20) Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_40] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_40] Select Operator [SEL_38] (rows=1 width=20) Output:["_col0","_col1","_col2"] Group By Operator [GBY_37] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 @@ -379,8 +379,8 @@ Stage-0 predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_52] (rows=3 width=16) Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_31] PartitionCols:_col0 Select Operator [SEL_29] (rows=18 width=84) Output:["_col0","_col1"] @@ -388,8 +388,8 @@ Stage-0 predicate:key is not null TableScan [TS_27] (rows=20 width=84) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_30] PartitionCols:_col0 Select Operator [SEL_26] (rows=1 width=101) Output:["_col0","_col1","_col2"] @@ -397,19 +397,19 @@ Stage-0 predicate:((_col1 + _col4) >= 0) Merge Join Operator [MERGEJOIN_51] (rows=1 width=101) Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Select Operator [SEL_20] (rows=1 width=89) Output:["_col0","_col1"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] Select Operator [SEL_17] (rows=1 width=105) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_16] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_14] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -417,19 +417,19 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) TableScan [TS_11] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] PartitionCols:_col0 Select Operator [SEL_9] (rows=1 width=97) Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] Select Operator [SEL_6] (rows=1 width=97) Output:["_col0","_col1","_col2"] Group By Operator [GBY_5] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=2 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -445,28 +445,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap + Vertex 7 llap File Output Operator [FS_39] Select Operator [SEL_38] (rows=1 width=20) Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_37] Group By Operator [GBY_34] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_33] PartitionCols:_col0, _col1 Group By Operator [GBY_32] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 @@ -476,8 +476,8 @@ Stage-0 predicate:(((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) Merge Join Operator [MERGEJOIN_48] (rows=3 width=16) Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] PartitionCols:_col0 Select Operator [SEL_26] (rows=18 width=84) Output:["_col0","_col1"] @@ -485,8 +485,8 @@ Stage-0 predicate:((c_int > 0) and key is not null) TableScan [TS_24] (rows=20 width=84) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_27] PartitionCols:_col0 Select Operator [SEL_23] (rows=1 width=101) Output:["_col0","_col1","_col2"] @@ -494,19 +494,19 @@ Stage-0 predicate:((_col1 + _col4) >= 0) Merge Join Operator [MERGEJOIN_47] (rows=1 width=101) Conds:RS_19._col0=RS_20._col0(Left Outer),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col0 Select Operator [SEL_9] (rows=1 width=97) Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] Select Operator [SEL_6] (rows=1 width=105) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_5] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -514,15 +514,15 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col0 Select Operator [SEL_17] (rows=1 width=89) Output:["_col0","_col1"] Group By Operator [GBY_16] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_14] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float @@ -538,24 +538,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_31] Select Operator [SEL_30] (rows=1 width=20) Output:["_col0","_col1","_col2"] Group By Operator [GBY_29] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] PartitionCols:_col0, _col1 Group By Operator [GBY_27] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 @@ -565,26 +565,19 @@ Stage-0 predicate:(((_col1 + _col4) >= 2) and ((_col1 > 0) or (_col6 >= 0))) Merge Join Operator [MERGEJOIN_36] (rows=4 width=20) Conds:RS_21._col0=RS_22._col0(Right Outer),RS_21._col0=RS_23._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_19] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_21] PartitionCols:_col0 Select Operator [SEL_9] (rows=1 width=97) Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] Select Operator [SEL_6] (rows=1 width=105) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_5] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -592,15 +585,15 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] PartitionCols:_col0 Select Operator [SEL_17] (rows=1 width=89) Output:["_col0","_col1"] Group By Operator [GBY_16] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_14] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float @@ -608,6 +601,13 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) TableScan [TS_11] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] + PartitionCols:_col0 + Select Operator [SEL_20] (rows=20 width=84) + Output:["_col0","_col1"] + TableScan [TS_19] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int PREHOOK: type: QUERY @@ -616,28 +616,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 llap + Vertex 6 llap File Output Operator [FS_37] Select Operator [SEL_36] (rows=1 width=20) Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_35] Group By Operator [GBY_33] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_32] PartitionCols:_col0, _col1 Group By Operator [GBY_31] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 @@ -647,8 +647,8 @@ Stage-0 predicate:(((_col1 + _col4) >= 0) and ((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) Merge Join Operator [MERGEJOIN_42] (rows=4 width=20) Conds:RS_25._col0=RS_26._col0(Outer),RS_25._col0=RS_27._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_27] PartitionCols:_col0 Select Operator [SEL_24] (rows=20 width=84) Output:["_col0","_col1"] @@ -656,19 +656,19 @@ Stage-0 predicate:(c_int > 0) TableScan [TS_22] (rows=20 width=84) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0 Select Operator [SEL_9] (rows=1 width=97) Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] Select Operator [SEL_6] (rows=1 width=105) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_5] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -676,19 +676,19 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_26] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_26] PartitionCols:_col0 Select Operator [SEL_20] (rows=1 width=89) Output:["_col0","_col1"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] Select Operator [SEL_17] (rows=1 width=105) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_16] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_14] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -704,24 +704,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_33] Select Operator [SEL_32] (rows=1 width=20) Output:["_col0","_col1","_col2"] Group By Operator [GBY_31] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_30] PartitionCols:_col0, _col1 Group By Operator [GBY_29] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 @@ -731,17 +731,8 @@ Stage-0 predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_43] (rows=3 width=16) Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=18 width=84) - predicate:key is not null - TableScan [TS_21] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_24] PartitionCols:_col0 Select Operator [SEL_20] (rows=1 width=101) Output:["_col0","_col1","_col2"] @@ -749,15 +740,15 @@ Stage-0 predicate:((_col1 + _col4) >= 0) Merge Join Operator [MERGEJOIN_42] (rows=1 width=101) Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0 Select Operator [SEL_6] (rows=1 width=97) Output:["_col0","_col1","_col2"] Group By Operator [GBY_5] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -765,15 +756,15 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_17] PartitionCols:_col0 Select Operator [SEL_14] (rows=1 width=89) Output:["_col0","_col1"] Group By Operator [GBY_13] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_11] (rows=1 width=93) Output:["_col0","_col1","_col2"],keys:key, c_int, c_float @@ -781,6 +772,15 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) TableScan [TS_8] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] + PartitionCols:_col0 + Select Operator [SEL_23] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_41] (rows=18 width=84) + predicate:key is not null + TableScan [TS_21] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] PREHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc PREHOOK: type: QUERY @@ -807,56 +807,56 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_26] Select Operator [SEL_25] (rows=3 width=87) Output:["_col0"] - <-Union 3 [SIMPLE_EDGE] - <-Reducer 2 [CONTAINS] llap + <-Union 3 [SORT_PARTITION_EDGE] + <-Vertex 2 [CONTAINS] llap Reduce Output Operator [RS_24] Select Operator [SEL_5] (rows=1 width=87) Output:["_col0"] Group By Operator [GBY_4] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Group By Operator [GBY_2] (rows=1 width=8) Output:["_col0"],aggregations:["count(key)"] Select Operator [SEL_1] (rows=20 width=80) Output:["key"] TableScan [TS_0] (rows=20 width=80) default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 6 [CONTAINS] llap + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_24] Select Operator [SEL_12] (rows=1 width=87) Output:["_col0"] Group By Operator [GBY_11] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] + <-Vertex 5 [PARTITION_EDGE] llap + PARTITION [RS_10] Group By Operator [GBY_9] (rows=1 width=8) Output:["_col0"],aggregations:["count(key)"] Select Operator [SEL_8] (rows=20 width=80) Output:["key"] TableScan [TS_7] (rows=20 width=80) default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 8 [CONTAINS] llap + <-Vertex 8 [CONTAINS] llap Reduce Output Operator [RS_24] Select Operator [SEL_21] (rows=1 width=87) Output:["_col0"] Group By Operator [GBY_20] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_19] + <-Vertex 7 [PARTITION_EDGE] llap + PARTITION [RS_19] Group By Operator [GBY_18] (rows=1 width=8) Output:["_col0"],aggregations:["count(key)"] Select Operator [SEL_17] (rows=20 width=80) @@ -879,26 +879,26 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) +Vertex 9 <- Union 3 (CONTAINS), Vertex 8 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_31] Select Operator [SEL_30] (rows=1 width=95) Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] Group By Operator [GBY_27] (rows=1 width=95) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Reducer 2 [CONTAINS] llap + <-Union 3 [SORT_PARTITION_EDGE] + <-Vertex 2 [CONTAINS] llap Reduce Output Operator [RS_26] PartitionCols:_col0 Group By Operator [GBY_25] (rows=1 width=95) @@ -907,15 +907,15 @@ Stage-0 Output:["_col0"] Group By Operator [GBY_4] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Group By Operator [GBY_2] (rows=1 width=8) Output:["_col0"],aggregations:["count(key)"] Select Operator [SEL_1] (rows=20 width=80) Output:["key"] TableScan [TS_0] (rows=20 width=80) default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 7 [CONTAINS] llap + <-Vertex 7 [CONTAINS] llap Reduce Output Operator [RS_26] PartitionCols:_col0 Group By Operator [GBY_25] (rows=1 width=95) @@ -924,15 +924,15 @@ Stage-0 Output:["_col0"] Group By Operator [GBY_11] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] + <-Vertex 6 [PARTITION_EDGE] llap + PARTITION [RS_10] Group By Operator [GBY_9] (rows=1 width=8) Output:["_col0"],aggregations:["count(key)"] Select Operator [SEL_8] (rows=20 width=80) Output:["key"] TableScan [TS_7] (rows=20 width=80) default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 9 [CONTAINS] llap + <-Vertex 9 [CONTAINS] llap Reduce Output Operator [RS_26] PartitionCols:_col0 Group By Operator [GBY_25] (rows=1 width=95) @@ -941,8 +941,8 @@ Stage-0 Output:["_col0"] Group By Operator [GBY_20] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 8 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_19] + <-Vertex 8 [PARTITION_EDGE] llap + PARTITION [RS_19] Group By Operator [GBY_18] (rows=1 width=8) Output:["_col0"],aggregations:["count(key)"] Select Operator [SEL_17] (rows=20 width=80) @@ -957,18 +957,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=18 width=85) Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=6 width=85) Output:["_col0"] @@ -976,8 +976,8 @@ Stage-0 predicate:(UDFToDouble(key) >= 1.0) TableScan [TS_0] (rows=20 width=80) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=6 width=85) Output:["_col0"] @@ -993,27 +993,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=100 width=8) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_11] (rows=100 width=8) Conds:RS_4._col0=RS_5._col0(Left Outer),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0 Select Operator [SEL_1] (rows=20 width=84) Output:["_col0","_col1"] TableScan [TS_0] (rows=20 width=84) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:_col0 Select Operator [SEL_3] (rows=20 width=84) Output:["_col0","_col1"] @@ -1027,27 +1027,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=100 width=8) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_9] (rows=100 width=8) Conds:RS_4._col0=RS_5._col0(Outer),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0 Select Operator [SEL_1] (rows=20 width=84) Output:["_col0","_col1"] TableScan [TS_0] (rows=20 width=84) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:_col0 Select Operator [SEL_3] (rows=20 width=84) Output:["_col0","_col1"] @@ -1061,20 +1061,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_14] Select Operator [SEL_13] (rows=291 width=101) Output:["_col0","_col1","_col2","_col3","_col4"] Merge Join Operator [MERGEJOIN_24] (rows=291 width=101) Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Select Operator [SEL_2] (rows=18 width=87) Output:["_col0","_col1","_col2"] @@ -1082,8 +1082,8 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col0 Select Operator [SEL_5] (rows=18 width=84) Output:["_col0","_col1"] @@ -1091,8 +1091,8 @@ Stage-0 predicate:key is not null TableScan [TS_3] (rows=20 width=84) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Select Operator [SEL_8] (rows=18 width=84) Output:["_col0","_col1"] @@ -1108,20 +1108,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_14] Select Operator [SEL_13] (rows=291 width=178) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_24] (rows=291 width=178) Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col0","_col1","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Select Operator [SEL_2] (rows=18 width=84) Output:["_col0","_col1"] @@ -1129,8 +1129,8 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=20 width=84) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col0 Select Operator [SEL_5] (rows=18 width=80) Output:["_col0"] @@ -1138,8 +1138,8 @@ Stage-0 predicate:key is not null TableScan [TS_3] (rows=20 width=80) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Select Operator [SEL_8] (rows=18 width=84) Output:["_col0","_col1"] @@ -1155,14 +1155,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_19] Select Operator [SEL_18] (rows=36 width=101) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -1170,24 +1170,15 @@ Stage-0 predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_28] (rows=36 width=101) Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=18 width=84) - predicate:key is not null - TableScan [TS_11] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col0 Filter Operator [FIL_9] (rows=10 width=182) predicate:(((_col1 + _col4) = 2) and ((_col4 + 1) = 2)) Merge Join Operator [MERGEJOIN_27] (rows=40 width=182) Conds:RS_6._col0=RS_7._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] @@ -1195,8 +1186,8 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=9 width=89) Output:["_col0","_col1"] @@ -1204,6 +1195,15 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] + PartitionCols:_col0 + Select Operator [SEL_13] (rows=18 width=84) + Output:["_col0","_col1"] + Filter Operator [FIL_26] (rows=18 width=84) + predicate:key is not null + TableScan [TS_11] (rows=20 width=84) + default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 right outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) PREHOOK: type: QUERY @@ -1212,13 +1212,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_14] Select Operator [SEL_13] (rows=50 width=101) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -1226,8 +1226,8 @@ Stage-0 predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)) and ((_col4 + 1) = 2)) Merge Join Operator [MERGEJOIN_19] (rows=200 width=101) Conds:RS_8._col0=RS_9._col0(Right Outer),RS_8._col0=RS_10._col0(Right Outer),Output:["_col1","_col2","_col3","_col4","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0 Select Operator [SEL_2] (rows=10 width=93) Output:["_col0","_col1","_col2"] @@ -1235,8 +1235,8 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Select Operator [SEL_5] (rows=10 width=89) Output:["_col0","_col1"] @@ -1244,8 +1244,8 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col0 Select Operator [SEL_7] (rows=20 width=84) Output:["_col0","_col1"] @@ -1259,27 +1259,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_10] Limit [LIM_9] (rows=1 width=97) Number of rows:1 Select Operator [SEL_8] (rows=10 width=97) Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] Select Operator [SEL_5] (rows=10 width=97) Output:["_col0","_col1","_col2"] Group By Operator [GBY_4] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_2] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -1295,26 +1295,26 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_15] Limit [LIM_14] (rows=1 width=20) Number of rows:1 Select Operator [SEL_13] (rows=5 width=20) Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] Group By Operator [GBY_10] (rows=5 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0, _col1 Group By Operator [GBY_8] (rows=5 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 @@ -1322,8 +1322,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_4] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_2] (rows=10 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -1339,14 +1339,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_13] Limit [LIM_12] (rows=5 width=85) Number of rows:5 @@ -1354,16 +1354,16 @@ Stage-0 Number of rows:5 Select Operator [SEL_9] (rows=5 width=85) Output:["_col0"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_8] Limit [LIM_7] (rows=5 width=85) Number of rows:5 Limit [LIM_5] (rows=5 width=85) Number of rows:5 Select Operator [SEL_4] (rows=5 width=85) Output:["_col0"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Limit [LIM_2] (rows=5 width=85) Number of rows:5 Select Operator [SEL_1] (rows=20 width=80) @@ -1378,34 +1378,34 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_13] Limit [LIM_12] (rows=5 width=89) Number of rows:5 Select Operator [SEL_11] (rows=5 width=89) Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] Limit [LIM_8] (rows=5 width=89) Number of rows:5 Select Operator [SEL_7] (rows=5 width=89) Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] Limit [LIM_4] (rows=5 width=89) Number of rows:5 Select Operator [SEL_3] (rows=20 width=84) Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] Select Operator [SEL_1] (rows=20 width=84) Output:["_col0","_col1"] TableScan [TS_0] (rows=20 width=84) @@ -1418,33 +1418,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 7 llap + Vertex 7 llap File Output Operator [FS_49] Limit [LIM_48] (rows=1 width=20) Number of rows:5 Select Operator [SEL_47] (rows=1 width=20) Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_46] Select Operator [SEL_44] (rows=1 width=20) Output:["_col0","_col1","_col2"] Group By Operator [GBY_43] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_42] PartitionCols:_col0, _col1 Group By Operator [GBY_41] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 @@ -1454,8 +1454,8 @@ Stage-0 predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_61] (rows=3 width=16) Conds:RS_36._col0=RS_37._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_37] PartitionCols:_col0 Select Operator [SEL_35] (rows=18 width=84) Output:["_col0","_col1"] @@ -1463,8 +1463,8 @@ Stage-0 predicate:key is not null TableScan [TS_33] (rows=20 width=84) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_36] PartitionCols:_col0 Select Operator [SEL_32] (rows=1 width=101) Output:["_col0","_col1","_col2"] @@ -1472,8 +1472,8 @@ Stage-0 predicate:((_col1 + _col4) >= 0) Merge Join Operator [MERGEJOIN_60] (rows=2 width=101) Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:_col0 Filter Operator [FIL_26] (rows=2 width=105) predicate:_col0 is not null @@ -1481,14 +1481,14 @@ Stage-0 Number of rows:5 Select Operator [SEL_23] (rows=3 width=105) Output:["_col0","_col1"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] Select Operator [SEL_20] (rows=3 width=105) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_19] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_18] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_17] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -1496,8 +1496,8 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) TableScan [TS_14] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] PartitionCols:_col0 Filter Operator [FIL_12] (rows=2 width=97) predicate:_col0 is not null @@ -1505,14 +1505,14 @@ Stage-0 Number of rows:5 Select Operator [SEL_9] (rows=3 width=97) Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] Select Operator [SEL_6] (rows=3 width=97) Output:["_col0","_col1","_col2"] Group By Operator [GBY_5] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=3 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -1528,20 +1528,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Select Operator [SEL_11] (rows=9 width=4) Output:["_col0"] Merge Join Operator [MERGEJOIN_17] (rows=9 width=4) Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1"] @@ -1549,8 +1549,8 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Group By Operator [GBY_7] (rows=5 width=85) Output:["_col0"],keys:_col0 @@ -1568,20 +1568,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_18] Select Operator [SEL_17] (rows=16 width=93) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_28] (rows=16 width=93) Conds:RS_13._col0=RS_14._col0(Left Semi),RS_13._col0=RS_15._col0(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] PartitionCols:_col0 Select Operator [SEL_2] (rows=9 width=93) Output:["_col0","_col1","_col2"] @@ -1589,8 +1589,8 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col0 Group By Operator [GBY_10] (rows=3 width=85) Output:["_col0"],keys:_col0 @@ -1600,8 +1600,8 @@ Stage-0 predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) TableScan [TS_3] (rows=20 width=88) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] PartitionCols:_col0 Group By Operator [GBY_12] (rows=6 width=85) Output:["_col0"],keys:_col0 @@ -1619,37 +1619,37 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 llap + Vertex 6 llap File Output Operator [FS_41] Select Operator [SEL_40] (rows=1 width=101) Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_39] Select Operator [SEL_38] (rows=1 width=101) Output:["_col0","_col1","_col2"] Group By Operator [GBY_37] (rows=1 width=101) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=1 width=101) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 Merge Join Operator [MERGEJOIN_51] (rows=1 width=93) Conds:RS_30._col0=RS_31._col0(Left Semi),RS_30._col0=RS_32._col0(Left Semi),Output:["_col0","_col1"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_32] PartitionCols:_col0 Group By Operator [GBY_29] (rows=3 width=85) Output:["_col0"],keys:_col0 @@ -1659,13 +1659,13 @@ Stage-0 predicate:(UDFToDouble(key) > 0.0) TableScan [TS_23] (rows=20 width=80) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_30] PartitionCols:_col0 Select Operator [SEL_10] (rows=1 width=93) Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] Select Operator [SEL_8] (rows=1 width=101) Output:["_col0","_col1","_col2"] Filter Operator [FIL_7] (rows=1 width=101) @@ -1674,8 +1674,8 @@ Stage-0 Output:["_col1","_col2","_col3"] Group By Operator [GBY_5] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_3] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -1683,21 +1683,21 @@ Stage-0 predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0)) and (UDFToDouble(key) > 0.0)) TableScan [TS_0] (rows=20 width=88) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_31] PartitionCols:_col0 Group By Operator [GBY_27] (rows=1 width=85) Output:["_col0"],keys:_col0 Select Operator [SEL_21] (rows=1 width=85) Output:["_col0"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] Select Operator [SEL_18] (rows=1 width=93) Output:["_col0","_col1"] Group By Operator [GBY_17] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_15] (rows=1 width=101) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float @@ -1756,30 +1756,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_15] Group By Operator [GBY_13] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_12] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_12] Group By Operator [GBY_11] (rows=1 width=8) Output:["_col0"],aggregations:["count('2014')"] Merge Join Operator [MERGEJOIN_18] (rows=400 width=8) Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_6] Select Operator [SEL_2] (rows=20 width=88) TableScan [TS_0] (rows=20 width=21) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Vertex 4 [PARTITION_EDGE] llap + PARTITION [RS_7] Select Operator [SEL_5] (rows=20 width=88) TableScan [TS_3] (rows=20 width=21) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE @@ -1803,16 +1803,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_29] Select Operator [SEL_28] (rows=1 width=178) Output:["_col0","_col1"] @@ -1820,15 +1820,15 @@ Stage-0 predicate:_col3 is null Merge Join Operator [MERGEJOIN_37] (rows=500 width=182) Conds:RS_24._col1=RS_25._col0(Left Outer),Output:["_col0","_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_24] PartitionCols:_col1 Select Operator [SEL_1] (rows=500 width=178) Output:["_col0","_col1"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0 Select Operator [SEL_23] (rows=56 width=95) Output:["_col0","_col1"] @@ -1838,8 +1838,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_17] (rows=83 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=83 width=178) Output:["_col0","_col1"],keys:_col2, _col0 @@ -1847,8 +1847,8 @@ Stage-0 Output:["_col2","_col0"] Merge Join Operator [MERGEJOIN_36] (rows=166 width=178) Conds:RS_11._col1=RS_12._col0(Inner),Output:["_col0","_col2"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col1 Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] @@ -1856,13 +1856,13 @@ Stage-0 predicate:(value > 'val_2') TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_9] (rows=214 width=91) Output:["_col0"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0 Group By Operator [GBY_7] (rows=214 width=91) Output:["_col0"],keys:value @@ -1890,17 +1890,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_33] Select Operator [SEL_32] (rows=1 width=178) Output:["_col0","_col1"] @@ -1908,13 +1908,13 @@ Stage-0 predicate:_col4 is null Merge Join Operator [MERGEJOIN_41] (rows=250 width=182) Conds:RS_28._col0, _col1=RS_29._col0, _col1(Left Outer),Output:["_col0","_col1","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] PartitionCols:_col0, _col1 Group By Operator [GBY_4] (rows=250 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0, _col1 Group By Operator [GBY_2] (rows=250 width=178) Output:["_col0","_col1"],keys:key, value @@ -1922,22 +1922,22 @@ Stage-0 Output:["key","value"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:_col0, _col1 Select Operator [SEL_27] (rows=1 width=182) Output:["_col0","_col1","_col2"] Group By Operator [GBY_26] (rows=1 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0, _col1 Group By Operator [GBY_24] (rows=1 width=178) Output:["_col0","_col1"],keys:_col2, _col3 Merge Join Operator [MERGEJOIN_40] (rows=1 width=178) Conds:RS_20._col0, _col1=RS_21._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col0, _col1 Select Operator [SEL_8] (rows=166 width=178) Output:["_col0","_col1"] @@ -1945,15 +1945,15 @@ Stage-0 predicate:(value > 'val_12') TableScan [TS_6] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_21] PartitionCols:_col0, _col1 Group By Operator [GBY_18] (rows=250 width=178) Output:["_col0","_col1"],keys:_col0, _col1 Group By Operator [GBY_13] (rows=250 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0, _col1 Group By Operator [GBY_11] (rows=250 width=178) Output:["_col0","_col1"],keys:key, value @@ -1991,40 +1991,40 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_23] Merge Join Operator [MERGEJOIN_33] (rows=1 width=178) Conds:RS_19._col0, _col1=RS_20._col0, _col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col0, _col1 Select Operator [SEL_1] (rows=500 width=178) Output:["_col0","_col1"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"} - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col0, _col1 Group By Operator [GBY_17] (rows=1 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0, _col1 Group By Operator [GBY_15] (rows=1 width=178) Output:["_col0","_col1"],keys:_col2, _col3 Merge Join Operator [MERGEJOIN_32] (rows=1 width=178) Conds:RS_11._col0, _col1=RS_12._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0, _col1 Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] @@ -2032,13 +2032,13 @@ Stage-0 predicate:(value > 'val_9') TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=250 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=250 width=178) Output:["_col0","_col1"],keys:key, value @@ -2066,40 +2066,40 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_23] Merge Join Operator [MERGEJOIN_33] (rows=1 width=178) Conds:RS_19._col0, _col1=RS_20._col0, _col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col0, _col1 Select Operator [SEL_1] (rows=500 width=178) Output:["_col0","_col1"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col0, _col1 Group By Operator [GBY_17] (rows=1 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0, _col1 Group By Operator [GBY_15] (rows=1 width=178) Output:["_col0","_col1"],keys:_col2, _col3 Merge Join Operator [MERGEJOIN_32] (rows=1 width=178) Conds:RS_11._col0, _col1=RS_12._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0, _col1 Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] @@ -2107,13 +2107,13 @@ Stage-0 predicate:(value > 'val_9') TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=250 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=250 width=178) Output:["_col0","_col1"],keys:key, value @@ -2131,19 +2131,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_14] Merge Join Operator [MERGEJOIN_19] (rows=166 width=178) Conds:RS_10._col0=RS_11._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col0 Select Operator [SEL_2] (rows=166 width=178) Output:["_col0","_col1"] @@ -2151,13 +2151,13 @@ Stage-0 predicate:(key > '9') TableScan [TS_0] (rows=500 width=178) default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Group By Operator [GBY_8] (rows=69 width=87) Output:["_col0"],keys:KEY._col0 - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Group By Operator [GBY_6] (rows=69 width=87) Output:["_col0"],keys:key @@ -2179,46 +2179,37 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_46] Select Operator [SEL_45] (rows=5 width=8) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_67] (rows=5 width=8) Conds:RS_42._col1, _col4=RS_43._col0, _col1(Inner),Output:["_col0","_col3"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_42] PartitionCols:_col1, _col4 Merge Join Operator [MERGEJOIN_64] (rows=5 width=16) Conds:RS_39._col0=RS_40._col1(Inner),Output:["_col0","_col1","_col3","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_40] - PartitionCols:_col1 - Select Operator [SEL_9] (rows=17 width=16) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_60] (rows=17 width=16) - predicate:((l_linenumber = 1) and l_partkey is not null) - TableScan [TS_7] (rows=100 width=16) - default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_39] PartitionCols:_col0 Group By Operator [GBY_5] (rows=50 width=4) Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0 Group By Operator [GBY_3] (rows=50 width=4) Output:["_col0"],keys:l_partkey @@ -2226,55 +2217,46 @@ Stage-0 predicate:l_partkey is not null TableScan [TS_0] (rows=100 width=4) default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_43] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_40] + PartitionCols:_col1 + Select Operator [SEL_9] (rows=17 width=16) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_60] (rows=17 width=16) + predicate:((l_linenumber = 1) and l_partkey is not null) + TableScan [TS_7] (rows=100 width=16) + default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_43] PartitionCols:_col0, _col1 Group By Operator [GBY_37] (rows=4 width=8) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=4 width=8) Output:["_col0","_col1"],keys:_col0, _col3 Merge Join Operator [MERGEJOIN_66] (rows=14 width=8) Conds:RS_31._col1=RS_32._col0(Inner),Output:["_col0","_col3"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=14 width=95) - Output:["_col0","_col1"] - Filter Operator [FIL_61] (rows=14 width=96) - predicate:(l_shipmode = 'AIR') - TableScan [TS_10] (rows=100 width=96) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] + <-Vertex 12 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_32] PartitionCols:_col0 Group By Operator [GBY_29] (rows=3 width=4) Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] PartitionCols:_col0 Group By Operator [GBY_27] (rows=3 width=4) Output:["_col0"],keys:_col2 Merge Join Operator [MERGEJOIN_65] (rows=34 width=4) Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col2"] - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=100 width=8) - Output:["_col0","_col1"] - Filter Operator [FIL_63] (rows=100 width=8) - predicate:l_partkey is not null - TableScan [TS_20] (rows=100 width=8) - default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey","l_linenumber"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Group By Operator [GBY_18] (rows=50 width=4) Output:["_col0"],keys:KEY._col0 - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_17] PartitionCols:_col0 Group By Operator [GBY_16] (rows=50 width=4) Output:["_col0"],keys:l_partkey @@ -2282,6 +2264,24 @@ Stage-0 predicate:l_partkey is not null TableScan [TS_13] (rows=100 width=4) default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] + <-Vertex 13 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_24] + PartitionCols:_col0 + Select Operator [SEL_22] (rows=100 width=8) + Output:["_col0","_col1"] + Filter Operator [FIL_63] (rows=100 width=8) + predicate:l_partkey is not null + TableScan [TS_20] (rows=100 width=8) + default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey","l_linenumber"] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_31] + PartitionCols:_col1 + Select Operator [SEL_12] (rows=14 width=95) + Output:["_col0","_col1"] + Filter Operator [FIL_61] (rows=14 width=96) + predicate:(l_shipmode = 'AIR') + TableScan [TS_10] (rows=100 width=96) + default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] PREHOOK: query: explain select key, value, count(*) from src_cbo b @@ -2298,37 +2298,37 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_36] Merge Join Operator [MERGEJOIN_49] (rows=34 width=186) Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_32] PartitionCols:_col2 Filter Operator [FIL_42] (rows=83 width=186) predicate:_col2 is not null Group By Operator [GBY_16] (rows=83 width=186) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_15] PartitionCols:_col0, _col1 Group By Operator [GBY_14] (rows=83 width=186) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 Merge Join Operator [MERGEJOIN_48] (rows=166 width=178) Conds:RS_10._col0=RS_11._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col0 Select Operator [SEL_2] (rows=166 width=178) Output:["_col0","_col1"] @@ -2336,13 +2336,13 @@ Stage-0 predicate:(key > '8') TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Group By Operator [GBY_8] (rows=69 width=87) Output:["_col0"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Group By Operator [GBY_6] (rows=69 width=87) Output:["_col0"],keys:key @@ -2350,13 +2350,13 @@ Stage-0 predicate:(key > '8') TableScan [TS_3] (rows=500 width=87) default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_33] PartitionCols:_col0 Group By Operator [GBY_30] (rows=34 width=8) Output:["_col0"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:_col0 Group By Operator [GBY_28] (rows=34 width=8) Output:["_col0"],keys:_col1 @@ -2366,8 +2366,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_24] (rows=69 width=95) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Group By Operator [GBY_22] (rows=69 width=95) Output:["_col0","_col1"],aggregations:["count()"],keys:key @@ -2391,28 +2391,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_23] Merge Join Operator [MERGEJOIN_28] (rows=6 width=227) Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col1 Select Operator [SEL_6] (rows=13 width=227) Output:["_col0","_col1","_col2"] Group By Operator [GBY_5] (rows=13 width=227) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1 Group By Operator [GBY_3] (rows=13 width=295) Output:["_col0","_col1","_col2"],aggregations:["avg(p_size)"],keys:p_name, p_mfgr @@ -2420,13 +2420,13 @@ Stage-0 predicate:p_name is not null TableScan [TS_0] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col0 Group By Operator [GBY_17] (rows=13 width=184) Output:["_col0"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=13 width=184) Output:["_col0"],keys:_col0 @@ -2438,8 +2438,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_9] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:p_mfgr TableScan [TS_7] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] @@ -2461,60 +2461,60 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_26] Select Operator [SEL_25] (rows=500 width=178) Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_24] Select Operator [SEL_23] (rows=500 width=178) Output:["_col0","_col1"] Filter Operator [FIL_22] (rows=500 width=198) predicate:((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) Merge Join Operator [MERGEJOIN_31] (rows=500 width=198) Conds:RS_19._col0=RS_20._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_30] (rows=500 width=194) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_16] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_16] Select Operator [SEL_1] (rows=500 width=178) Output:["_col0","_col1"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] + <-Vertex 6 [PARTITION_EDGE] llap + PARTITION [RS_17] Group By Operator [GBY_7] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 5 [PARTITION_EDGE] llap + PARTITION [RS_6] Group By Operator [GBY_5] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count()","count(key)"] Filter Operator [FIL_28] (rows=166 width=87) predicate:(key > '2') TableScan [TS_2] (rows=500 width=87) default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col0 Select Operator [SEL_15] (rows=69 width=91) Output:["_col0","_col1"] Group By Operator [GBY_14] (rows=69 width=87) Output:["_col0"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] PartitionCols:_col0 Group By Operator [GBY_12] (rows=69 width=87) Output:["_col0"],keys:key @@ -2542,22 +2542,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_54] Select Operator [SEL_53] (rows=13 width=223) Output:["_col0","_col1","_col2"] @@ -2565,77 +2565,77 @@ Stage-0 predicate:CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END Merge Join Operator [MERGEJOIN_76] (rows=26 width=243) Conds:RS_49._col0, _col1=RS_50._col3, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_50] + <-Vertex 12 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_50] PartitionCols:_col3, _col1 Merge Join Operator [MERGEJOIN_75] (rows=2 width=223) Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_42] PartitionCols:_col0 Select Operator [SEL_35] (rows=4 width=223) Output:["_col0","_col1","_col2"] Group By Operator [GBY_34] (rows=4 width=219) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_33] PartitionCols:_col0, _col1 Group By Operator [GBY_32] (rows=4 width=219) Output:["_col0","_col1"],keys:_col1, _col2 Merge Join Operator [MERGEJOIN_74] (rows=8 width=219) Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=8 width=219) - Output:["_col0","_col1"] - Filter Operator [FIL_69] (rows=8 width=223) - predicate:(p_size < 10) - TableScan [TS_19] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 14 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:_col0 Group By Operator [GBY_26] (rows=5 width=98) Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 13 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0 Group By Operator [GBY_24] (rows=5 width=98) Output:["_col0"],keys:p_mfgr TableScan [TS_22] (rows=26 width=98) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_43] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] + PartitionCols:_col0 + Select Operator [SEL_21] (rows=8 width=219) + Output:["_col0","_col1"] + Filter Operator [FIL_69] (rows=8 width=223) + predicate:(p_size < 10) + TableScan [TS_19] (rows=26 width=223) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + <-Vertex 16 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_43] PartitionCols:_col0 Group By Operator [GBY_40] (rows=13 width=121) Output:["_col0"],keys:KEY._col0 - <-Map 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] + <-Vertex 15 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_39] PartitionCols:_col0 Group By Operator [GBY_38] (rows=13 width=121) Output:["_col0"],keys:p_name TableScan [TS_36] (rows=26 width=121) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_49] PartitionCols:_col0, _col1 Merge Join Operator [MERGEJOIN_73] (rows=26 width=239) Conds:RS_46._col1=RS_47._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_46] PartitionCols:_col1 Select Operator [SEL_1] (rows=26 width=223) Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_47] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_47] PartitionCols:_col0 Group By Operator [GBY_17] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col1)"],keys:_col2 @@ -2643,8 +2643,8 @@ Stage-0 Output:["_col2","_col1"] Merge Join Operator [MERGEJOIN_72] (rows=8 width=219) Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col1","_col2"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Select Operator [SEL_4] (rows=8 width=219) Output:["_col0","_col1"] @@ -2652,13 +2652,13 @@ Stage-0 predicate:(p_size < 10) TableScan [TS_2] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_9] (rows=5 width=98) Output:["_col0"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0 Group By Operator [GBY_7] (rows=5 width=98) Output:["_col0"],keys:p_mfgr @@ -2684,70 +2684,70 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (PARTITION_EDGE) +Vertex 8 <- Vertex 7 (PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_36] Select Operator [SEL_35] (rows=26 width=125) Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_34] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_34] Select Operator [SEL_33] (rows=26 width=125) Output:["_col0","_col1"] Filter Operator [FIL_32] (rows=26 width=145) predicate:((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2))) Merge Join Operator [MERGEJOIN_42] (rows=26 width=145) Conds:RS_29.UDFToDouble(_col1)=RS_30._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:UDFToDouble(_col1) Merge Join Operator [MERGEJOIN_41] (rows=26 width=141) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_26] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_26] Select Operator [SEL_1] (rows=26 width=125) Output:["_col0","_col1"] TableScan [TS_0] (rows=26 width=125) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_27] + <-Vertex 6 [PARTITION_EDGE] llap + PARTITION [RS_27] Group By Operator [GBY_12] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] Group By Operator [GBY_7] (rows=1 width=8) Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 5 [PARTITION_EDGE] llap + PARTITION [RS_6] Group By Operator [GBY_5] (rows=1 width=76) Output:["_col0"],aggregations:["avg(p_size)"] Filter Operator [FIL_38] (rows=8 width=4) predicate:(p_size < 10) TableScan [TS_2] (rows=26 width=4) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_30] PartitionCols:_col0 Select Operator [SEL_25] (rows=1 width=12) Output:["_col0","_col1"] Group By Operator [GBY_24] (rows=1 width=8) Output:["_col0"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Group By Operator [GBY_22] (rows=1 width=8) Output:["_col0"],keys:_col0 Group By Operator [GBY_19] (rows=1 width=8) Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_18] + <-Vertex 7 [PARTITION_EDGE] llap + PARTITION [RS_18] Group By Operator [GBY_17] (rows=1 width=76) Output:["_col0"],aggregations:["avg(p_size)"] Filter Operator [FIL_40] (rows=8 width=4) @@ -2778,60 +2778,60 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 12 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_83] Select Operator [SEL_82] (rows=2 width=106) Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_81] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_81] Select Operator [SEL_80] (rows=2 width=106) Output:["_col0","_col1"] Filter Operator [FIL_79] (rows=2 width=126) predicate:CASE WHEN ((_col3 = 0)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END Merge Join Operator [MERGEJOIN_108] (rows=5 width=126) Conds:RS_76._col0, _col1=RS_77._col3, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_77] + <-Vertex 17 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_77] PartitionCols:_col3, _col1 Merge Join Operator [MERGEJOIN_107] (rows=1 width=110) Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_69] + <-Vertex 16 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_69] PartitionCols:_col0 Select Operator [SEL_57] (rows=1 width=110) Output:["_col0","_col1","_col2"] Group By Operator [GBY_56] (rows=1 width=106) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] + <-Vertex 15 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_55] PartitionCols:_col0, _col1 Group By Operator [GBY_54] (rows=1 width=106) Output:["_col0","_col1"],keys:_col0, _col3 Merge Join Operator [MERGEJOIN_106] (rows=1 width=106) Conds:RS_50._col1=RS_51._col0(Inner),Output:["_col0","_col3"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_50] + <-Vertex 14 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_50] PartitionCols:_col1 Select Operator [SEL_38] (rows=1 width=114) Output:["_col0","_col1"] @@ -2839,20 +2839,20 @@ Stage-0 predicate:((_col2 - _col1) > 600.0) Group By Operator [GBY_36] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] + <-Vertex 13 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_35] PartitionCols:_col0 Group By Operator [GBY_34] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr TableScan [TS_32] (rows=26 width=106) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 20 [SIMPLE_EDGE] llap - SHUFFLE [RS_51] + <-Vertex 20 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_51] PartitionCols:_col0 Group By Operator [GBY_48] (rows=2 width=8) Output:["_col0"],keys:KEY._col0 - <-Reducer 19 [SIMPLE_EDGE] llap - SHUFFLE [RS_47] + <-Vertex 19 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_47] PartitionCols:_col0 Group By Operator [GBY_46] (rows=2 width=8) Output:["_col0"],keys:_col1 @@ -2860,8 +2860,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_43] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] + <-Vertex 18 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_42] PartitionCols:_col0 Group By Operator [GBY_41] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr @@ -2869,32 +2869,32 @@ Stage-0 Output:["p_mfgr","p_retailprice"] TableScan [TS_39] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_70] + <-Vertex 22 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_70] PartitionCols:_col0 Group By Operator [GBY_67] (rows=5 width=98) Output:["_col0"],keys:_col0 Group By Operator [GBY_62] (rows=5 width=98) Output:["_col0"],keys:KEY._col0 - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_61] + <-Vertex 21 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_61] PartitionCols:_col0 Group By Operator [GBY_60] (rows=5 width=98) Output:["_col0"],keys:p_mfgr TableScan [TS_58] (rows=26 width=98) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_76] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_76] PartitionCols:_col0, _col1 Merge Join Operator [MERGEJOIN_105] (rows=5 width=122) Conds:RS_73._col1=RS_74._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_73] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_73] PartitionCols:_col1 Group By Operator [GBY_4] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0 Group By Operator [GBY_2] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr @@ -2902,13 +2902,13 @@ Stage-0 Output:["p_mfgr","p_retailprice"] TableScan [TS_0] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_74] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_74] PartitionCols:_col0 Group By Operator [GBY_30] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:_col0 Group By Operator [GBY_28] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col0)"],keys:_col3 @@ -2916,13 +2916,13 @@ Stage-0 Output:["_col3","_col0"] Merge Join Operator [MERGEJOIN_104] (rows=1 width=106) Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col0","_col3"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 12 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0 Group By Operator [GBY_22] (rows=2 width=8) Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_21] PartitionCols:_col0 Group By Operator [GBY_20] (rows=2 width=8) Output:["_col0"],keys:_col1 @@ -2930,8 +2930,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_17] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr @@ -2939,8 +2939,8 @@ Stage-0 Output:["p_mfgr","p_retailprice"] TableScan [TS_13] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_24] PartitionCols:_col1 Select Operator [SEL_12] (rows=1 width=114) Output:["_col0","_col1"] @@ -2948,8 +2948,8 @@ Stage-0 predicate:((_col2 - _col1) > 600.0) Group By Operator [GBY_10] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Group By Operator [GBY_8] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr @@ -2965,13 +2965,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_4] (rows=20 width=52) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] @@ -2979,8 +2979,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] Select Operator [SEL_2] (rows=20 width=459) Output:["_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:0 TableScan [TS_0] (rows=20 width=7) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] @@ -2992,13 +2992,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_4] (rows=20 width=52) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] @@ -3006,8 +3006,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] Select Operator [SEL_2] (rows=20 width=459) Output:["_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:0 TableScan [TS_0] (rows=20 width=7) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] @@ -3019,13 +3019,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_4] (rows=20 width=64) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] @@ -3033,8 +3033,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] Select Operator [SEL_2] (rows=20 width=621) Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:key TableScan [TS_0] (rows=20 width=169) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int","key","value"] @@ -3046,13 +3046,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_4] (rows=25 width=179) Output:["_col0","_col1","_col2"] @@ -3060,8 +3060,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] Select Operator [SEL_2] (rows=25 width=443) Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:key TableScan [TS_0] (rows=25 width=175) default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -3081,35 +3081,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_20] Group By Operator [GBY_18] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_17] Group By Operator [GBY_16] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] Select Operator [SEL_14] (rows=14 width=94) Output:["_col0","_col1"] Group By Operator [GBY_13] (rows=14 width=94) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_11] (rows=14 width=94) Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 Merge Join Operator [MERGEJOIN_25] (rows=60 width=86) Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=25 width=86) Output:["_col0"] @@ -3117,8 +3117,8 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=25 width=86) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=500 width=87) Output:["_col0"] @@ -3142,35 +3142,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_20] Group By Operator [GBY_18] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_17] Group By Operator [GBY_16] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] Select Operator [SEL_14] (rows=14 width=94) Output:["_col0","_col1"] Group By Operator [GBY_13] (rows=14 width=94) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_11] (rows=14 width=94) Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 Merge Join Operator [MERGEJOIN_25] (rows=60 width=86) Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=25 width=86) Output:["_col0"] @@ -3178,8 +3178,8 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=25 width=86) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=500 width=87) Output:["_col0"] @@ -3203,34 +3203,34 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_20] Group By Operator [GBY_18] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_17] Group By Operator [GBY_16] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] Select Operator [SEL_14] (rows=14 width=94) Output:["_col0","_col1"] Group By Operator [GBY_13] (rows=14 width=94) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col0 Group By Operator [GBY_11] (rows=14 width=94) Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 Map Join Operator [MAPJOIN_25] (rows=60 width=86) Conds:RS_6._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Map 1 [BROADCAST_EDGE] llap + <-Vertex 1 [BROADCAST_EDGE] llap BROADCAST [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=25 width=86) @@ -3261,35 +3261,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_22] Group By Operator [GBY_20] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_19] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_19] Group By Operator [GBY_18] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] Select Operator [SEL_16] (rows=12 width=94) Output:["_col0","_col1"] Group By Operator [GBY_15] (rows=12 width=94) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col0 Group By Operator [GBY_13] (rows=12 width=94) Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 Merge Join Operator [MERGEJOIN_27] (rows=25 width=86) Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col0 Select Operator [SEL_2] (rows=25 width=86) Output:["_col0"] @@ -3297,8 +3297,8 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=25 width=86) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Group By Operator [GBY_7] (rows=205 width=87) Output:["_col0"],keys:_col0 @@ -3340,18 +3340,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Group By Operator [GBY_4] (rows=2 width=19) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col2)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:_col0 Group By Operator [GBY_2] (rows=4 width=19) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(DISTINCT b)","count(DISTINCT c)","sum(d)"],keys:a, b, c @@ -3367,18 +3367,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_5] Group By Operator [GBY_3] (rows=2 width=19) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:a Select Operator [SEL_1] (rows=4 width=19) Output:["a","b","c","d"] @@ -3459,18 +3459,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Group By Operator [GBY_4] (rows=1 width=8) Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Select Operator [SEL_1] (rows=5 width=6) TableScan [TS_0] (rows=5 width=6) default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE @@ -3482,18 +3482,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Group By Operator [GBY_4] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Select Operator [SEL_1] (rows=5 width=6) Output:["_col0","_col1"] TableScan [TS_0] (rows=5 width=6) @@ -3531,18 +3531,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Group By Operator [GBY_4] (rows=1 width=8) Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Select Operator [SEL_1] (rows=5 width=6) TableScan [TS_0] (rows=5 width=6) default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE @@ -3554,18 +3554,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Group By Operator [GBY_4] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_3] Select Operator [SEL_1] (rows=5 width=6) Output:["_col0","_col1"] TableScan [TS_0] (rows=5 width=6) @@ -3594,24 +3594,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Merge Join Operator [MERGEJOIN_9] (rows=250000 width=87) Conds:(Inner),Output:["_col0"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_4] Select Operator [SEL_1] (rows=500 width=87) Output:["_col0"] TableScan [TS_0] (rows=500 width=87) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_5] Select Operator [SEL_3] (rows=500 width=4) TableScan [TS_2] (rows=500 width=10) default@src,src2,Tbl:COMPLETE,Col:COMPLETE @@ -3639,8 +3639,8 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -3650,21 +3650,21 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_8] table:{"name:":"default.nzhang_CTAS1"} Limit [LIM_7] (rows=10 width=178) Number of rows:10 Select Operator [SEL_6] (rows=10 width=178) Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] Limit [LIM_4] (rows=10 width=178) Number of rows:10 Select Operator [SEL_3] (rows=500 width=178) Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] Select Operator [SEL_1] (rows=500 width=178) Output:["_col0","_col1"] TableScan [TS_0] (rows=500 width=178) @@ -3692,8 +3692,8 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -3703,21 +3703,21 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_8] table:{"name:":"default.nzhang_ctas3"} Limit [LIM_7] (rows=10 width=192) Number of rows:10 Select Operator [SEL_6] (rows=10 width=192) Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] Limit [LIM_4] (rows=10 width=192) Number of rows:10 Select Operator [SEL_3] (rows=500 width=192) Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] Select Operator [SEL_1] (rows=500 width=192) Output:["_col0","_col1"] TableScan [TS_0] (rows=500 width=178) @@ -3782,31 +3782,31 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_12] Select Operator [SEL_11] (rows=27556 width=356) Output:["_col0","_col1","_col2","_col3"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] Merge Join Operator [MERGEJOIN_15] (rows=27556 width=356) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_6] Select Operator [SEL_2] (rows=166 width=178) Output:["_col0","_col1"] Filter Operator [FIL_13] (rows=166 width=178) predicate:(key < 10) TableScan [TS_0] (rows=500 width=178) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Vertex 4 [PARTITION_EDGE] llap + PARTITION [RS_7] Select Operator [SEL_5] (rows=166 width=178) Output:["_col0","_col1"] Filter Operator [FIL_14] (rows=166 width=178) @@ -3837,25 +3837,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=3 width=9) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) Conds:RS_2.key=RS_3.value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:value TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -3867,34 +3867,34 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_11] Select Operator [SEL_10] (rows=6 width=9) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_21] (rows=6 width=9) Conds:RS_4.key=RS_6.value(Inner),RS_4.key=RS_8.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:key Filter Operator [FIL_18] (rows=3 width=8) predicate:key is not null TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:value Filter Operator [FIL_19] (rows=3 width=8) predicate:value is not null TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:key Filter Operator [FIL_20] (rows=3 width=8) predicate:key is not null @@ -3908,30 +3908,30 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=6 width=9) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_9] (rows=6 width=9) Conds:RS_3.key=RS_4.value(Inner),RS_3.key=RS_5.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:value TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:key TableScan [TS_2] (rows=3 width=8) default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -3943,34 +3943,34 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_11] Select Operator [SEL_10] (rows=6 width=9) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_15] (rows=6 width=9) Conds:RS_4.key, value=RS_6.value, key(Inner),RS_4.key, value=RS_8.key, value(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:key, value Filter Operator [FIL_12] (rows=3 width=8) predicate:value is not null TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:value, key Filter Operator [FIL_13] (rows=3 width=8) predicate:key is not null TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:key, value Filter Operator [FIL_14] (rows=3 width=8) predicate:value is not null @@ -3984,30 +3984,30 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=6 width=9) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Merge Join Operator [MERGEJOIN_9] (rows=6 width=9) Conds:RS_3.key, value=RS_4.value, key(Inner),RS_3.key, value=RS_5.key, value(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key, value TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:value, key TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:key, value TableScan [TS_2] (rows=3 width=8) default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4019,25 +4019,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=3 width=9) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) Conds:RS_2.key=RS_3.value(Left Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:value TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4049,25 +4049,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=3 width=9) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) Conds:RS_2.key=RS_3.value(Right Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:value TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4079,25 +4079,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=3 width=9) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) Conds:RS_2.key=RS_3.value(Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:value TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4109,25 +4109,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=3 width=9) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) Conds:RS_2.key=RS_3.value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=3 width=8) default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:value TableScan [TS_1] (rows=3 width=8) default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4205,25 +4205,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=28 width=7) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) Conds:RS_2.key=RS_3.key(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=26 width=7) default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key TableScan [TS_1] (rows=26 width=7) default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4235,25 +4235,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=28 width=7) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) Conds:RS_2.key, value=RS_3.key, value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key, value TableScan [TS_0] (rows=26 width=7) default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key, value TableScan [TS_1] (rows=26 width=7) default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4265,25 +4265,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=28 width=7) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) Conds:RS_2.key=RS_3.key(Right Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=26 width=7) default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key TableScan [TS_1] (rows=26 width=7) default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4295,25 +4295,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=28 width=7) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) Conds:RS_2.key=RS_3.key(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=26 width=7) default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key TableScan [TS_1] (rows=26 width=7) default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4325,25 +4325,25 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_6] Select Operator [SEL_5] (rows=28 width=7) Output:["_col0","_col1","_col2","_col3"] Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) Conds:RS_2.key=RS_3.key(Left Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:key TableScan [TS_0] (rows=26 width=7) default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key TableScan [TS_1] (rows=26 width=7) default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -4409,18 +4409,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=2 width=15) Conds:RS_8._col1=RS_9._col0(Left Semi),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col1 Select Operator [SEL_2] (rows=1 width=13) Output:["_col0","_col1"] @@ -4428,8 +4428,8 @@ Stage-0 predicate:id is not null TableScan [TS_0] (rows=1 width=13) default@sales,sales,Tbl:COMPLETE,Col:NONE,Output:["name","id"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Group By Operator [GBY_7] (rows=2 width=14) Output:["_col0"],keys:_col0 @@ -4463,17 +4463,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_16] Map Join Operator [MAPJOIN_26] (rows=805 width=10) Conds:MAPJOIN_25._col1=RS_13._col0(Inner),Output:["_col0"] - <-Map 3 [BROADCAST_EDGE] llap + <-Vertex 3 [BROADCAST_EDGE] llap BROADCAST [RS_13] PartitionCols:_col0 Select Operator [SEL_8] (rows=166 width=10) @@ -4484,7 +4484,7 @@ Stage-0 default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] <-Map Join Operator [MAPJOIN_25] (rows=732 width=10) Conds:SEL_2._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 2 [BROADCAST_EDGE] llap + <-Vertex 2 [BROADCAST_EDGE] llap BROADCAST [RS_10] PartitionCols:_col0 Select Operator [SEL_5] (rows=25 width=7) @@ -4507,17 +4507,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_16] Map Join Operator [MAPJOIN_26] (rows=805 width=10) Conds:MAPJOIN_25._col1=RS_13._col0(Inner),Output:["_col0"] - <-Map 3 [BROADCAST_EDGE] llap + <-Vertex 3 [BROADCAST_EDGE] llap BROADCAST [RS_13] PartitionCols:_col0 Select Operator [SEL_8] (rows=166 width=10) @@ -4528,7 +4528,7 @@ Stage-0 default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] <-Map Join Operator [MAPJOIN_25] (rows=732 width=10) Conds:SEL_2._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 2 [BROADCAST_EDGE] llap + <-Vertex 2 [BROADCAST_EDGE] llap BROADCAST [RS_10] PartitionCols:_col0 Select Operator [SEL_5] (rows=25 width=7) @@ -4567,14 +4567,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_9] Select Operator [SEL_7] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -4582,15 +4582,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_5] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=231) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] @@ -4612,15 +4612,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_16] Select Operator [SEL_14] (rows=29 width=227) Output:["_col0","_col1","_col2","_col3"] @@ -4628,19 +4628,19 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_12] (rows=29 width=223) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col2 PTF Operator [PTF_10] (rows=29 width=223) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_9] (rows=29 width=223) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col2 Map Join Operator [MAPJOIN_21] (rows=29 width=223) Conds:FIL_19.p_partkey=RS_5.p_partkey(Inner),Output:["_col1","_col2","_col5"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_5] PartitionCols:p_partkey Filter Operator [FIL_20] (rows=26 width=4) @@ -4675,14 +4675,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_9] Select Operator [SEL_7] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -4690,15 +4690,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_5] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=231) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] @@ -4726,14 +4726,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_9] Select Operator [SEL_7] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] @@ -4741,15 +4741,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_5] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] @@ -4779,15 +4779,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_14] Select Operator [SEL_12] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] @@ -4795,13 +4795,13 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] Group By Operator [GBY_8] (rows=26 width=223) Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Group By Operator [GBY_6] (rows=26 width=223) Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:rand() Select Operator [SEL_4] (rows=26 width=491) Output:["_col1","_col2","_col5"] @@ -4809,8 +4809,8 @@ Stage-0 Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] @@ -4832,17 +4832,17 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_11] Map Join Operator [MAPJOIN_16] (rows=29 width=619) Conds:FIL_14._col0=RS_8.p_partkey(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 3 [BROADCAST_EDGE] llap + <-Vertex 3 [BROADCAST_EDGE] llap BROADCAST [RS_8] PartitionCols:p_partkey Filter Operator [FIL_15] (rows=26 width=4) @@ -4855,8 +4855,8 @@ Stage-0 Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_3] (rows=26 width=887) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=619) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_brand","p_comment","p_container","p_mfgr","p_name","p_partkey","p_retailprice","p_size","p_type"] @@ -4878,14 +4878,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_10] Select Operator [SEL_8] (rows=26 width=227) Output:["_col0","_col1","_col2","_col3"] @@ -4893,15 +4893,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}] Select Operator [SEL_6] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:_col2 PTF Operator [PTF_4] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}}] Select Operator [SEL_3] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:p_mfgr PTF Operator [PTF_1] (rows=26 width=223) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST, p_size DESC NULLS LAST","partition by:":"p_mfgr"}}] @@ -4929,14 +4929,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_10] Select Operator [SEL_8] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -4944,15 +4944,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_6] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:_col2 PTF Operator [PTF_4] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_3] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:p_mfgr PTF Operator [PTF_1] (rows=26 width=231) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST","partition by:":"p_mfgr"}}] @@ -4980,14 +4980,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_9] Select Operator [SEL_7] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -4995,15 +4995,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_5] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=231) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] @@ -5031,15 +5031,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_13] Select Operator [SEL_11] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -5047,15 +5047,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_9] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col2 PTF Operator [PTF_7] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_6] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:_col2 PTF Operator [PTF_4] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] @@ -5063,8 +5063,8 @@ Stage-0 Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=231) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] @@ -5098,14 +5098,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_10] Select Operator [SEL_7] (rows=26 width=235) Output:["_col0","_col1","_col2","_col3"] @@ -5113,15 +5113,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_5] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=231) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] @@ -5153,14 +5153,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_15] Select Operator [SEL_13] (rows=29 width=259) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] @@ -5168,12 +5168,12 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_11] (rows=29 width=767) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col2 Map Join Operator [MAPJOIN_20] (rows=29 width=231) Conds:FIL_18._col0=RS_8.p_partkey(Inner),Output:["_col1","_col2","_col5","_col7"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_8] PartitionCols:p_partkey Filter Operator [FIL_19] (rows=26 width=4) @@ -5186,8 +5186,8 @@ Stage-0 Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_3] (rows=26 width=503) Output:["_col0","_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=235) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_partkey","p_retailprice","p_size"] @@ -5292,10 +5292,10 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-4 Stats-Aggr Operator @@ -5305,7 +5305,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_9] table:{"name:":"default.part_4"} Select Operator [SEL_7] (rows=26 width=239) @@ -5314,19 +5314,19 @@ Stage-4 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_5] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=499) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=499) Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=231) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_20] table:{"name:":"default.part_5"} Select Operator [SEL_17] (rows=26 width=247) @@ -5335,8 +5335,8 @@ Stage-4 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST","partition by:":"_col3"}] Select Operator [SEL_15] (rows=26 width=499) Output:["_col0","_col2","_col3","_col6"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:_col2 Select Operator [SEL_13] (rows=26 width=491) Output:["_col1","_col2","_col5","sum_window_0"] @@ -5344,8 +5344,8 @@ Stage-4 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_11] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col2 Please refer to the previous PTF Operator [PTF_3] Stage-5 @@ -5392,15 +5392,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_13] Select Operator [SEL_11] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -5408,15 +5408,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}] Select Operator [SEL_9] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_8] PartitionCols:_col2, _col1 PTF Operator [PTF_7] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] Select Operator [SEL_6] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:_col2, _col1 PTF Operator [PTF_4] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] @@ -5424,8 +5424,8 @@ Stage-0 Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] @@ -5467,16 +5467,16 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_15] Select Operator [SEL_13] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -5484,29 +5484,29 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_11] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col2 PTF Operator [PTF_9] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_8] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col2 PTF Operator [PTF_6] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] Select Operator [SEL_5] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2, _col1 PTF Operator [PTF_3] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_2] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr TableScan [TS_0] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] @@ -5544,15 +5544,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_12] Select Operator [SEL_10] (rows=26 width=239) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -5560,22 +5560,22 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] Select Operator [SEL_8] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col2 PTF Operator [PTF_6] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] Select Operator [SEL_5] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col2 PTF Operator [PTF_3] (rows=26 width=491) Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] Select Operator [SEL_2] (rows=26 width=491) Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_1] PartitionCols:p_mfgr, p_name TableScan [TS_0] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] @@ -5587,24 +5587,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_7] Group By Operator [GBY_5] (rows=500 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:_col0, _col1 Group By Operator [GBY_3] (rows=500 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:rand() Select Operator [SEL_1] (rows=500 width=178) Output:["key","value"] @@ -5709,7 +5709,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -5719,15 +5719,15 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_11] table:{"name:":"default.dest_j1"} Select Operator [SEL_9] (rows=1219 width=95) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_16] (rows=1219 width=178) Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=500 width=87) Output:["_col0"] @@ -5735,8 +5735,8 @@ Stage-3 predicate:key is not null TableScan [TS_0] (rows=500 width=87) default@src,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=500 width=178) Output:["_col0","_col1"] @@ -5772,33 +5772,33 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_14] Select Operator [SEL_13] (rows=3 width=33) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Map Join Operator [MAPJOIN_31] (rows=3 width=33) Conds:FIL_27.key=RS_7.key(Inner),RS_7.key=RS_9.key(Inner),RS_9.key=RS_11.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] - <-Map 2 [BROADCAST_EDGE] llap + <-Vertex 2 [BROADCAST_EDGE] llap BROADCAST [RS_7] PartitionCols:key Filter Operator [FIL_28] (rows=1 width=30) predicate:key is not null TableScan [TS_1] (rows=1 width=30) default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 3 [BROADCAST_EDGE] llap + <-Vertex 3 [BROADCAST_EDGE] llap BROADCAST [RS_9] PartitionCols:key Filter Operator [FIL_29] (rows=1 width=20) predicate:key is not null TableScan [TS_2] (rows=1 width=20) default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_11] PartitionCols:key Filter Operator [FIL_30] (rows=1 width=30) @@ -5825,33 +5825,33 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_14] Select Operator [SEL_13] (rows=3 width=33) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Map Join Operator [MAPJOIN_31] (rows=3 width=33) Conds:FIL_27.key=RS_7.key(Inner),RS_7.key=RS_9.key(Inner),RS_9.key=RS_11.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] - <-Map 2 [BROADCAST_EDGE] llap + <-Vertex 2 [BROADCAST_EDGE] llap BROADCAST [RS_7] PartitionCols:key Filter Operator [FIL_28] (rows=1 width=30) predicate:key is not null TableScan [TS_1] (rows=1 width=30) default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 3 [BROADCAST_EDGE] llap + <-Vertex 3 [BROADCAST_EDGE] llap BROADCAST [RS_9] PartitionCols:key Filter Operator [FIL_29] (rows=1 width=20) predicate:key is not null TableScan [TS_2] (rows=1 width=20) default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_11] PartitionCols:key Filter Operator [FIL_30] (rows=1 width=30) @@ -5870,28 +5870,28 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_13] Group By Operator [GBY_11] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_10] Group By Operator [GBY_9] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_8] PartitionCols:rand() Map Join Operator [MAPJOIN_18] (rows=550 width=87) Conds:RS_3.UDFToDouble(key)=FIL_17.(key + 1)(Inner),Output:["_col0","_col1","_col5"] - <-Map 1 [BROADCAST_EDGE] llap + <-Vertex 1 [BROADCAST_EDGE] llap BROADCAST [RS_3] PartitionCols:UDFToDouble(key) Filter Operator [FIL_16] (rows=1 width=30) @@ -5931,18 +5931,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=1219 width=356) Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=500 width=178) Output:["_col0","_col1"] @@ -5950,8 +5950,8 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=500 width=178) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=500 width=178) Output:["_col0","_col1"] @@ -5967,28 +5967,28 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_13] Group By Operator [GBY_11] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_10] Group By Operator [GBY_9] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_8] PartitionCols:rand() Map Join Operator [MAPJOIN_18] (rows=1 width=33) Conds:FIL_16.key=RS_5.val(Inner),Output:["_col0","_col6"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_5] PartitionCols:val Filter Operator [FIL_17] (rows=1 width=30) @@ -6007,30 +6007,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_16] Group By Operator [GBY_14] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_13] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_13] Group By Operator [GBY_12] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] PartitionCols:rand() Select Operator [SEL_9] (rows=1 width=33) Output:["_col0","_col1"] Map Join Operator [MAPJOIN_21] (rows=1 width=33) Conds:SEL_2._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=1 width=30) @@ -6053,28 +6053,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_16] Group By Operator [GBY_14] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_13] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_13] Group By Operator [GBY_12] (rows=1 width=8) Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] PartitionCols:rand() Map Join Operator [MAPJOIN_21] (rows=1 width=33) Conds:SEL_2._col0=RS_7._col0(Inner) - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=1 width=30) @@ -6097,30 +6097,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_14] Group By Operator [GBY_12] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_9] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_9] PartitionCols:rand() Select Operator [SEL_7] (rows=1 width=33) Output:["_col0","_col1","_col2"] Map Join Operator [MAPJOIN_17] (rows=1 width=33) Conds:SEL_1.UDFToDouble(_col0)=RS_5.(UDFToDouble(_col0) + 1.0)(Left Outer),Output:["_col0","_col1","_col2"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_5] PartitionCols:(UDFToDouble(_col0) + 1.0) Select Operator [SEL_3] (rows=1 width=30) @@ -6139,28 +6139,28 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_11] Group By Operator [GBY_9] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_8] Group By Operator [GBY_7] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_6] PartitionCols:rand() Map Join Operator [MAPJOIN_14] (rows=1 width=33) Conds:RS_2.UDFToDouble(key)=TS_1.(key + 1)(Right Outer),Output:["_col0","_col1","_col5"] - <-Map 1 [BROADCAST_EDGE] llap + <-Vertex 1 [BROADCAST_EDGE] llap BROADCAST [RS_2] PartitionCols:UDFToDouble(key) TableScan [TS_0] (rows=1 width=30) @@ -6175,34 +6175,34 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_11] Group By Operator [GBY_9] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_8] Group By Operator [GBY_7] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_6] PartitionCols:rand() Merge Join Operator [MERGEJOIN_12] (rows=1 width=33) Conds:RS_2.UDFToDouble(key)=RS_3.(key + 1)(Outer),Output:["_col0","_col1","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_2] PartitionCols:UDFToDouble(key) TableScan [TS_0] (rows=1 width=30) default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:(key + 1) TableScan [TS_1] (rows=1 width=30) default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] @@ -6214,28 +6214,28 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_11] Group By Operator [GBY_9] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_8] Group By Operator [GBY_7] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_6] PartitionCols:rand() Map Join Operator [MAPJOIN_14] (rows=1 width=33) Conds:TS_0.(key + 1)=RS_3.UDFToDouble(key)(Left Outer),Output:["_col0","_col6"] - <-Map 4 [BROADCAST_EDGE] llap + <-Vertex 4 [BROADCAST_EDGE] llap BROADCAST [RS_3] PartitionCols:UDFToDouble(key) TableScan [TS_1] (rows=1 width=30) diff --git ql/src/test/results/clientpositive/llap/explainuser_2.q.out ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 67f960a..7a20bae 100644 --- ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -177,35 +177,26 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_16] Select Operator [SEL_15] (rows=141 width=268) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_26] (rows=141 width=268) Conds:RS_12._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=500 width=178) - predicate:key is not null - TableScan [TS_6] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col3 Merge Join Operator [MERGEJOIN_25] (rows=58 width=177) Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col0","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_9] PartitionCols:_col0 Select Operator [SEL_2] (rows=500 width=369) Output:["_col0"] @@ -213,8 +204,8 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=500 width=91) default@srcpart,z,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_10] PartitionCols:_col1 Select Operator [SEL_5] (rows=25 width=175) Output:["_col0","_col1"] @@ -222,6 +213,15 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_3] (rows=25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_24] (rows=500 width=178) + predicate:key is not null + TableScan [TS_6] (rows=500 width=178) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: EXPLAIN select @@ -282,32 +282,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_55] Limit [LIM_54] (rows=24 width=285) Number of rows:100 Select Operator [SEL_53] (rows=24 width=285) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_52] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_52] Group By Operator [GBY_50] (rows=24 width=285) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_49] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_48] (rows=24 width=285) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2 @@ -315,20 +315,20 @@ Stage-0 Output:["_col12","_col20","_col2","_col13","_col21","_col3"] Merge Join Operator [MERGEJOIN_97] (rows=650 width=534) Conds:RS_44._col1, _col3=RS_45._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"] - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_45] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_45] PartitionCols:_col15, _col17 Select Operator [SEL_40] (rows=190 width=447) Output:["_col14","_col15","_col17","_col6","_col7"] Merge Join Operator [MERGEJOIN_96] (rows=190 width=447) Conds:RS_37._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_37] PartitionCols:_col6, _col4 Merge Join Operator [MERGEJOIN_94] (rows=40 width=352) Conds:RS_34._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] + <-Vertex 14 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_35] PartitionCols:_col1 Select Operator [SEL_17] (rows=2 width=180) Output:["_col1"] @@ -336,13 +336,13 @@ Stage-0 predicate:((key = 'src1key') and value is not null) TableScan [TS_15] (rows=25 width=175) default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_34] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_34] PartitionCols:_col3 Merge Join Operator [MERGEJOIN_93] (rows=40 width=352) Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] + <-Vertex 13 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_32] PartitionCols:_col0 Select Operator [SEL_14] (rows=2 width=178) Output:["_col0"] @@ -350,13 +350,13 @@ Stage-0 predicate:((value = 'd1value') and key is not null) TableScan [TS_12] (rows=500 width=178) default@src,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] + <-Vertex 8 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_31] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_92] (rows=40 width=352) Conds:RS_28._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] + <-Vertex 12 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_29] PartitionCols:_col3 Select Operator [SEL_11] (rows=8 width=531) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -364,8 +364,8 @@ Stage-0 predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null) TableScan [TS_9] (rows=85 width=534) default@ss,ss,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_28] PartitionCols:_col1 Select Operator [SEL_8] (rows=10 width=185) Output:["_col1"] @@ -373,13 +373,13 @@ Stage-0 predicate:((key = 'srcpartkey') and value is not null) TableScan [TS_6] (rows=2000 width=178) default@srcpart,srcpart,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_38] + <-Vertex 16 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_38] PartitionCols:_col4, _col2 Merge Join Operator [MERGEJOIN_95] (rows=19 width=356) Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] + <-Vertex 15 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_24] PartitionCols:_col0 Select Operator [SEL_20] (rows=8 width=531) Output:["_col0","_col2","_col3","_col4","_col5"] @@ -387,8 +387,8 @@ Stage-0 predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null) TableScan [TS_18] (rows=85 width=534) default@sr,sr,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"] - <-Map 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 17 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0 Select Operator [SEL_23] (rows=500 width=178) Output:["_col0"] @@ -396,13 +396,13 @@ Stage-0 predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) TableScan [TS_21] (rows=500 width=178) default@src,d2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_44] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_44] PartitionCols:_col1, _col3 Merge Join Operator [MERGEJOIN_91] (rows=414 width=269) Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_41] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_41] PartitionCols:_col0 Select Operator [SEL_2] (rows=170 width=356) Output:["_col0","_col1","_col2","_col3"] @@ -410,8 +410,8 @@ Stage-0 predicate:(v2 is not null and v3 is not null and k1 is not null) TableScan [TS_0] (rows=170 width=356) default@cs,cs,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v2","k3","v3"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] + <-Vertex 6 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_42] PartitionCols:_col0 Select Operator [SEL_5] (rows=500 width=178) Output:["_col0"] @@ -441,28 +441,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 6 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap + Vertex 7 llap File Output Operator [FS_59] Group By Operator [GBY_57] (rows=28 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] llap + <-Union 6 [SORT_PARTITION_EDGE] + <-Vertex 15 [CONTAINS] llap Reduce Output Operator [RS_56] PartitionCols:_col0, _col1 Group By Operator [GBY_55] (rows=28 width=177) @@ -471,38 +471,20 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_85] (rows=73 width=177) Conds:RS_48._col2=RS_49._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_81] (rows=500 width=87) - predicate:key is not null - TableScan [TS_42] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_48] + <-Vertex 14 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_48] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_84] (rows=30 width=177) Conds:RS_45._col1=RS_46._col1(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_39] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_45] + <-Vertex 13 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_45] PartitionCols:_col1 Select Operator [SEL_38] (rows=262 width=178) Output:["_col1"] Group By Operator [GBY_37] (rows=262 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] llap + <-Union 12 [SORT_PARTITION_EDGE] + <-Vertex 11 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=262 width=178) @@ -513,7 +495,7 @@ Stage-0 predicate:value is not null TableScan [TS_26] (rows=25 width=175) Output:["key","value"] - <-Map 16 [CONTAINS] llap + <-Vertex 16 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=262 width=178) @@ -524,7 +506,25 @@ Stage-0 predicate:value is not null TableScan [TS_29] (rows=500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] llap + <-Vertex 17 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_46] + PartitionCols:_col1 + Select Operator [SEL_41] (rows=25 width=175) + Output:["_col0","_col1"] + Filter Operator [FIL_80] (rows=25 width=175) + predicate:(key is not null and value is not null) + TableScan [TS_39] (rows=25 width=175) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Vertex 18 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_49] + PartitionCols:_col0 + Select Operator [SEL_44] (rows=500 width=87) + Output:["_col0"] + Filter Operator [FIL_81] (rows=500 width=87) + predicate:key is not null + TableScan [TS_42] (rows=500 width=87) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + <-Vertex 5 [CONTAINS] llap Reduce Output Operator [RS_56] PartitionCols:_col0, _col1 Group By Operator [GBY_55] (rows=28 width=177) @@ -533,8 +533,8 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_83] (rows=73 width=177) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 10 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Select Operator [SEL_18] (rows=500 width=87) Output:["_col0"] @@ -542,29 +542,20 @@ Stage-0 predicate:key is not null TableScan [TS_16] (rows=500 width=87) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_82] (rows=30 width=177) Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col1 Select Operator [SEL_12] (rows=262 width=178) Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=262 width=178) @@ -575,7 +566,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25 width=175) Output:["key","value"] - <-Map 8 [CONTAINS] llap + <-Vertex 8 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=262 width=178) @@ -586,6 +577,15 @@ Stage-0 predicate:value is not null TableScan [TS_3] (rows=500 width=178) Output:["key","value"] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] + PartitionCols:_col1 + Select Operator [SEL_15] (rows=25 width=175) + Output:["_col0","_col1"] + Filter Operator [FIL_76] (rows=25 width=175) + predicate:(key is not null and value is not null) + TableScan [TS_13] (rows=25 width=175) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: explain SELECT x.key, y.value @@ -616,40 +616,40 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 10 <- Union 2 (CONTAINS) -Map 13 <- Union 14 (CONTAINS) -Map 20 <- Union 14 (CONTAINS) -Map 21 <- Union 16 (CONTAINS) -Map 24 <- Union 25 (CONTAINS) -Map 33 <- Union 25 (CONTAINS) -Map 34 <- Union 27 (CONTAINS) -Map 35 <- Union 29 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) -Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 30 <- Union 29 (SIMPLE_EDGE) -Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 10 <- Union 2 (CONTAINS) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 6 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 14 (CONTAINS) +Vertex 21 <- Union 16 (CONTAINS) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 8 (CONTAINS), Vertex 31 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 33 <- Union 25 (CONTAINS) +Vertex 34 <- Union 27 (CONTAINS) +Vertex 35 <- Union 29 (CONTAINS) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 llap + Vertex 9 llap File Output Operator [FS_122] Group By Operator [GBY_120] (rows=107 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 32 [CONTAINS] llap + <-Union 8 [SORT_PARTITION_EDGE] + <-Vertex 32 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=107 width=177) @@ -658,49 +658,20 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_170] (rows=124 width=177) Conds:RS_111._col2=RS_112._col0(Inner),Output:["_col2","_col5"] - <-Map 37 [SIMPLE_EDGE] llap - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=178) - predicate:key is not null - TableScan [TS_105] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] llap - SHUFFLE [RS_111] + <-Vertex 31 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_111] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_169] (rows=51 width=86) Conds:RS_108._col1=RS_109._col1(Inner),Output:["_col2"] - <-Map 36 [SIMPLE_EDGE] llap - SHUFFLE [RS_109] - PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_102] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 30 [SIMPLE_EDGE] llap - SHUFFLE [RS_108] + <-Vertex 30 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_108] PartitionCols:_col1 Select Operator [SEL_101] (rows=440 width=178) Output:["_col1"] Group By Operator [GBY_100] (rows=440 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 29 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=440 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_94] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=500 width=178) - predicate:value is not null - TableScan [TS_92] (rows=500 width=178) - Output:["key","value"] - <-Reducer 28 [CONTAINS] llap + <-Union 29 [SORT_PARTITION_EDGE] + <-Vertex 28 [CONTAINS] llap Reduce Output Operator [RS_99] PartitionCols:_col0, _col1 Group By Operator [GBY_98] (rows=440 width=178) @@ -709,19 +680,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_90] (rows=381 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 27 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_84] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=178) - predicate:value is not null - TableScan [TS_82] (rows=500 width=178) - Output:["key","value"] - <-Reducer 26 [CONTAINS] llap + <-Union 27 [SORT_PARTITION_EDGE] + <-Vertex 26 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col0, _col1 Group By Operator [GBY_88] (rows=381 width=178) @@ -730,8 +690,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_80] (rows=262 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 24 [CONTAINS] llap + <-Union 25 [SORT_PARTITION_EDGE] + <-Vertex 24 [CONTAINS] llap Reduce Output Operator [RS_79] PartitionCols:_col0, _col1 Group By Operator [GBY_78] (rows=262 width=178) @@ -742,7 +702,7 @@ Stage-0 predicate:value is not null TableScan [TS_69] (rows=25 width=175) Output:["key","value"] - <-Map 33 [CONTAINS] llap + <-Vertex 33 [CONTAINS] llap Reduce Output Operator [RS_79] PartitionCols:_col0, _col1 Group By Operator [GBY_78] (rows=262 width=178) @@ -753,15 +713,55 @@ Stage-0 predicate:value is not null TableScan [TS_72] (rows=500 width=178) Output:["key","value"] - <-Reducer 7 [CONTAINS] llap + <-Vertex 34 [CONTAINS] llap + Reduce Output Operator [RS_89] + PartitionCols:_col0, _col1 + Group By Operator [GBY_88] (rows=381 width=178) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_84] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_161] (rows=500 width=178) + predicate:value is not null + TableScan [TS_82] (rows=500 width=178) + Output:["key","value"] + <-Vertex 35 [CONTAINS] llap + Reduce Output Operator [RS_99] + PartitionCols:_col0, _col1 + Group By Operator [GBY_98] (rows=440 width=178) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_94] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_162] (rows=500 width=178) + predicate:value is not null + TableScan [TS_92] (rows=500 width=178) + Output:["key","value"] + <-Vertex 36 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_109] + PartitionCols:_col1 + Select Operator [SEL_104] (rows=25 width=175) + Output:["_col0","_col1"] + Filter Operator [FIL_163] (rows=25 width=175) + predicate:(key is not null and value is not null) + TableScan [TS_102] (rows=25 width=175) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Vertex 37 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_112] + PartitionCols:_col0 + Select Operator [SEL_107] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_164] (rows=500 width=178) + predicate:key is not null + TableScan [TS_105] (rows=500 width=178) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Vertex 7 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=107 width=177) Output:["_col0","_col1"],keys:_col0, _col1 Group By Operator [GBY_67] (rows=90 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] llap + <-Union 6 [SORT_PARTITION_EDGE] + <-Vertex 19 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=90 width=177) @@ -770,49 +770,20 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_168] (rows=107 width=177) Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 23 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=178) - predicate:key is not null - TableScan [TS_52] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] + <-Vertex 18 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_58] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_167] (rows=44 width=86) Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] + <-Vertex 17 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_55] PartitionCols:_col1 Select Operator [SEL_48] (rows=381 width=178) Output:["_col1"] Group By Operator [GBY_47] (rows=381 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 16 [SIMPLE_EDGE] - <-Map 21 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500 width=178) - predicate:value is not null - TableScan [TS_39] (rows=500 width=178) - Output:["key","value"] - <-Reducer 15 [CONTAINS] llap + <-Union 16 [SORT_PARTITION_EDGE] + <-Vertex 15 [CONTAINS] llap Reduce Output Operator [RS_46] PartitionCols:_col0, _col1 Group By Operator [GBY_45] (rows=381 width=178) @@ -821,8 +792,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_37] (rows=262 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] llap + <-Union 14 [SORT_PARTITION_EDGE] + <-Vertex 13 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=262 width=178) @@ -833,7 +804,7 @@ Stage-0 predicate:value is not null TableScan [TS_26] (rows=25 width=175) Output:["key","value"] - <-Map 20 [CONTAINS] llap + <-Vertex 20 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=262 width=178) @@ -844,7 +815,36 @@ Stage-0 predicate:value is not null TableScan [TS_29] (rows=500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] llap + <-Vertex 21 [CONTAINS] llap + Reduce Output Operator [RS_46] + PartitionCols:_col0, _col1 + Group By Operator [GBY_45] (rows=381 width=178) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_41] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_156] (rows=500 width=178) + predicate:value is not null + TableScan [TS_39] (rows=500 width=178) + Output:["key","value"] + <-Vertex 22 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_56] + PartitionCols:_col1 + Select Operator [SEL_51] (rows=25 width=175) + Output:["_col0","_col1"] + Filter Operator [FIL_157] (rows=25 width=175) + predicate:(key is not null and value is not null) + TableScan [TS_49] (rows=25 width=175) + default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Vertex 23 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_59] + PartitionCols:_col0 + Select Operator [SEL_54] (rows=500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_158] (rows=500 width=178) + predicate:key is not null + TableScan [TS_52] (rows=500 width=178) + default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Vertex 5 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=90 width=177) @@ -853,8 +853,8 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_166] (rows=73 width=177) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 12 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Select Operator [SEL_18] (rows=500 width=178) Output:["_col0","_col1"] @@ -862,13 +862,13 @@ Stage-0 predicate:key is not null TableScan [TS_16] (rows=500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_165] (rows=30 width=86) Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_20] PartitionCols:_col1 Select Operator [SEL_15] (rows=25 width=175) Output:["_col0","_col1"] @@ -876,15 +876,15 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_13] (rows=25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] PartitionCols:_col1 Select Operator [SEL_12] (rows=262 width=178) Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=262 width=178) @@ -895,7 +895,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25 width=175) Output:["key","value"] - <-Map 10 [CONTAINS] llap + <-Vertex 10 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=262 width=178) @@ -920,19 +920,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_16] Select Operator [SEL_15] (rows=605 width=10) Output:["_col0","_col1","_col2"] Map Join Operator [MAPJOIN_26] (rows=605 width=10) Conds:MAPJOIN_25._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"] - <-Map 3 [BROADCAST_EDGE] llap + <-Vertex 3 [BROADCAST_EDGE] llap BROADCAST [RS_13] PartitionCols:_col0 Select Operator [SEL_8] (rows=500 width=10) @@ -943,7 +943,7 @@ Stage-0 default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_25] (rows=550 width=10) Conds:SEL_2._col0=RS_10._col1(Inner),Output:["_col0","_col3"] - <-Map 2 [BROADCAST_EDGE] llap + <-Vertex 2 [BROADCAST_EDGE] llap BROADCAST [RS_10] PartitionCols:_col1 Select Operator [SEL_5] (rows=25 width=7) @@ -1018,28 +1018,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 10 <- Map 9 (BROADCAST_EDGE) -Map 2 <- Map 1 (BROADCAST_EDGE) -Map 3 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 10 (BROADCAST_EDGE), Vertex 2 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_55] Limit [LIM_54] (rows=100 width=10) Number of rows:100 Select Operator [SEL_53] (rows=805 width=10) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_52] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_52] Group By Operator [GBY_50] (rows=805 width=10) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_49] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_48] (rows=1610 width=10) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2 @@ -1047,12 +1047,12 @@ Stage-0 Output:["_col12","_col20","_col2","_col13","_col21","_col3"] Map Join Operator [MAPJOIN_97] (rows=1610 width=10) Conds:RS_44._col1, _col3=SEL_40._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"] - <-Map 2 [BROADCAST_EDGE] llap + <-Vertex 2 [BROADCAST_EDGE] llap BROADCAST [RS_44] PartitionCols:_col1, _col3 Map Join Operator [MAPJOIN_91] (rows=275 width=10) Conds:RS_41._col0=SEL_5._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [BROADCAST_EDGE] llap + <-Vertex 1 [BROADCAST_EDGE] llap BROADCAST [RS_41] PartitionCols:_col0 Select Operator [SEL_2] (rows=170 width=34) @@ -1071,12 +1071,12 @@ Stage-0 Output:["_col14","_col15","_col17","_col6","_col7"] Map Join Operator [MAPJOIN_96] (rows=1464 width=10) Conds:MAPJOIN_94._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"] - <-Map 10 [BROADCAST_EDGE] llap + <-Vertex 10 [BROADCAST_EDGE] llap BROADCAST [RS_38] PartitionCols:_col4, _col2 Map Join Operator [MAPJOIN_95] (rows=275 width=10) Conds:RS_24._col0=SEL_23._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 9 [BROADCAST_EDGE] llap + <-Vertex 9 [BROADCAST_EDGE] llap BROADCAST [RS_24] PartitionCols:_col0 Select Operator [SEL_20] (rows=42 width=34) @@ -1093,7 +1093,7 @@ Stage-0 default@src,d2,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_94] (rows=1331 width=10) Conds:MAPJOIN_93._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 8 [BROADCAST_EDGE] llap + <-Vertex 8 [BROADCAST_EDGE] llap BROADCAST [RS_35] PartitionCols:_col1 Select Operator [SEL_17] (rows=12 width=7) @@ -1104,7 +1104,7 @@ Stage-0 default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_93] (rows=1210 width=10) Conds:MAPJOIN_92._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 7 [BROADCAST_EDGE] llap + <-Vertex 7 [BROADCAST_EDGE] llap BROADCAST [RS_32] PartitionCols:_col0 Select Operator [SEL_14] (rows=250 width=10) @@ -1115,7 +1115,7 @@ Stage-0 default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_92] (rows=1100 width=10) Conds:SEL_8._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 6 [BROADCAST_EDGE] llap + <-Vertex 6 [BROADCAST_EDGE] llap BROADCAST [RS_29] PartitionCols:_col3 Select Operator [SEL_11] (rows=42 width=34) @@ -1152,24 +1152,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 10 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Map 9 <- Union 10 (CONTAINS) -Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 13 (BROADCAST_EDGE), Vertex 14 (BROADCAST_EDGE) +Vertex 12 <- Union 10 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) +Vertex 9 <- Union 10 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_59] Group By Operator [GBY_57] (rows=550 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] llap + <-Union 4 [SORT_PARTITION_EDGE] + <-Vertex 11 [CONTAINS] llap Reduce Output Operator [RS_56] PartitionCols:_col0, _col1 Group By Operator [GBY_55] (rows=1100 width=10) @@ -1178,7 +1178,7 @@ Stage-0 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_85] (rows=550 width=10) Conds:MAPJOIN_84._col2=RS_49._col0(Inner),Output:["_col1","_col2"] - <-Map 14 [BROADCAST_EDGE] llap + <-Vertex 14 [BROADCAST_EDGE] llap BROADCAST [RS_49] PartitionCols:_col0 Select Operator [SEL_44] (rows=500 width=10) @@ -1189,7 +1189,7 @@ Stage-0 default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Map Join Operator [MAPJOIN_84] (rows=288 width=10) Conds:SEL_38._col1=RS_46._col1(Inner),Output:["_col1","_col2"] - <-Map 13 [BROADCAST_EDGE] llap + <-Vertex 13 [BROADCAST_EDGE] llap BROADCAST [RS_46] PartitionCols:_col1 Select Operator [SEL_41] (rows=25 width=7) @@ -1202,8 +1202,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_37] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 10 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap + <-Union 10 [SORT_PARTITION_EDGE] + <-Vertex 12 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -1214,7 +1214,7 @@ Stage-0 predicate:value is not null TableScan [TS_29] (rows=500 width=10) Output:["key","value"] - <-Map 9 [CONTAINS] llap + <-Vertex 9 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -1225,7 +1225,7 @@ Stage-0 predicate:value is not null TableScan [TS_26] (rows=25 width=7) Output:["key","value"] - <-Reducer 3 [CONTAINS] llap + <-Vertex 3 [CONTAINS] llap Reduce Output Operator [RS_56] PartitionCols:_col0, _col1 Group By Operator [GBY_55] (rows=1100 width=10) @@ -1234,7 +1234,7 @@ Stage-0 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_83] (rows=550 width=10) Conds:MAPJOIN_82._col2=RS_23._col0(Inner),Output:["_col1","_col2"] - <-Map 8 [BROADCAST_EDGE] llap + <-Vertex 8 [BROADCAST_EDGE] llap BROADCAST [RS_23] PartitionCols:_col0 Select Operator [SEL_18] (rows=500 width=10) @@ -1245,7 +1245,7 @@ Stage-0 default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Map Join Operator [MAPJOIN_82] (rows=288 width=10) Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col1","_col2"] - <-Map 7 [BROADCAST_EDGE] llap + <-Vertex 7 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 Select Operator [SEL_15] (rows=25 width=7) @@ -1258,8 +1258,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -1270,7 +1270,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25 width=7) Output:["key","value"] - <-Map 6 [CONTAINS] llap + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -1311,34 +1311,34 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 17 <- Union 14 (CONTAINS) -Map 20 <- Union 21 (CONTAINS) -Map 27 <- Union 21 (CONTAINS) -Map 28 <- Union 23 (CONTAINS) -Map 29 <- Union 25 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) -Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS) -Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS) -Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE), Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 18 (BROADCAST_EDGE), Vertex 19 (BROADCAST_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 17 <- Union 14 (CONTAINS) +Vertex 20 <- Union 21 (CONTAINS) +Vertex 22 <- Union 21 (SORT_PARTITION_EDGE), Union 23 (CONTAINS) +Vertex 24 <- Union 23 (SORT_PARTITION_EDGE), Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 6 (CONTAINS), Vertex 30 (BROADCAST_EDGE), Vertex 31 (BROADCAST_EDGE) +Vertex 27 <- Union 21 (CONTAINS) +Vertex 28 <- Union 23 (CONTAINS) +Vertex 29 <- Union 25 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Union 6 (CONTAINS) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap + Vertex 7 llap File Output Operator [FS_122] Group By Operator [GBY_120] (rows=550 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 26 [CONTAINS] llap + <-Union 6 [SORT_PARTITION_EDGE] + <-Vertex 26 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=1100 width=10) @@ -1347,7 +1347,7 @@ Stage-0 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_170] (rows=550 width=10) Conds:MAPJOIN_169._col2=RS_112._col0(Inner),Output:["_col2","_col5"] - <-Map 31 [BROADCAST_EDGE] llap + <-Vertex 31 [BROADCAST_EDGE] llap BROADCAST [RS_112] PartitionCols:_col0 Select Operator [SEL_107] (rows=500 width=10) @@ -1358,7 +1358,7 @@ Stage-0 default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_169] (rows=484 width=10) Conds:SEL_101._col1=RS_109._col1(Inner),Output:["_col2"] - <-Map 30 [BROADCAST_EDGE] llap + <-Vertex 30 [BROADCAST_EDGE] llap BROADCAST [RS_109] PartitionCols:_col1 Select Operator [SEL_104] (rows=25 width=7) @@ -1371,19 +1371,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_100] (rows=440 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 29 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_94] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=500 width=10) - predicate:value is not null - TableScan [TS_92] (rows=500 width=10) - Output:["key","value"] - <-Reducer 24 [CONTAINS] llap + <-Union 25 [SORT_PARTITION_EDGE] + <-Vertex 24 [CONTAINS] llap Reduce Output Operator [RS_99] PartitionCols:_col0, _col1 Group By Operator [GBY_98] (rows=881 width=10) @@ -1392,19 +1381,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_90] (rows=381 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 23 [SIMPLE_EDGE] - <-Map 28 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_84] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=10) - predicate:value is not null - TableScan [TS_82] (rows=500 width=10) - Output:["key","value"] - <-Reducer 22 [CONTAINS] llap + <-Union 23 [SORT_PARTITION_EDGE] + <-Vertex 22 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col0, _col1 Group By Operator [GBY_88] (rows=762 width=10) @@ -1413,8 +1391,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_80] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 21 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap + <-Union 21 [SORT_PARTITION_EDGE] + <-Vertex 20 [CONTAINS] llap Reduce Output Operator [RS_79] PartitionCols:_col0, _col1 Group By Operator [GBY_78] (rows=525 width=10) @@ -1425,7 +1403,7 @@ Stage-0 predicate:value is not null TableScan [TS_69] (rows=25 width=7) Output:["key","value"] - <-Map 27 [CONTAINS] llap + <-Vertex 27 [CONTAINS] llap Reduce Output Operator [RS_79] PartitionCols:_col0, _col1 Group By Operator [GBY_78] (rows=525 width=10) @@ -1436,15 +1414,37 @@ Stage-0 predicate:value is not null TableScan [TS_72] (rows=500 width=10) Output:["key","value"] - <-Reducer 5 [CONTAINS] llap + <-Vertex 28 [CONTAINS] llap + Reduce Output Operator [RS_89] + PartitionCols:_col0, _col1 + Group By Operator [GBY_88] (rows=762 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_84] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_161] (rows=500 width=10) + predicate:value is not null + TableScan [TS_82] (rows=500 width=10) + Output:["key","value"] + <-Vertex 29 [CONTAINS] llap + Reduce Output Operator [RS_99] + PartitionCols:_col0, _col1 + Group By Operator [GBY_98] (rows=881 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_94] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_162] (rows=500 width=10) + predicate:value is not null + TableScan [TS_92] (rows=500 width=10) + Output:["key","value"] + <-Vertex 5 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=1100 width=10) Output:["_col0","_col1"],keys:_col0, _col1 Group By Operator [GBY_67] (rows=550 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] llap + <-Union 4 [SORT_PARTITION_EDGE] + <-Vertex 15 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=1100 width=10) @@ -1453,7 +1453,7 @@ Stage-0 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_168] (rows=550 width=10) Conds:MAPJOIN_167._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 19 [BROADCAST_EDGE] llap + <-Vertex 19 [BROADCAST_EDGE] llap BROADCAST [RS_59] PartitionCols:_col0 Select Operator [SEL_54] (rows=500 width=10) @@ -1464,7 +1464,7 @@ Stage-0 default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_167] (rows=419 width=10) Conds:SEL_48._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 18 [BROADCAST_EDGE] llap + <-Vertex 18 [BROADCAST_EDGE] llap BROADCAST [RS_56] PartitionCols:_col1 Select Operator [SEL_51] (rows=25 width=7) @@ -1477,19 +1477,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_47] (rows=381 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 13 [CONTAINS] llap + <-Union 14 [SORT_PARTITION_EDGE] + <-Vertex 13 [CONTAINS] llap Reduce Output Operator [RS_46] PartitionCols:_col0, _col1 Group By Operator [GBY_45] (rows=762 width=10) @@ -1498,8 +1487,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_37] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] llap + <-Union 12 [SORT_PARTITION_EDGE] + <-Vertex 11 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -1510,7 +1499,7 @@ Stage-0 predicate:value is not null TableScan [TS_26] (rows=25 width=7) Output:["key","value"] - <-Map 16 [CONTAINS] llap + <-Vertex 16 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -1521,7 +1510,18 @@ Stage-0 predicate:value is not null TableScan [TS_29] (rows=500 width=10) Output:["key","value"] - <-Reducer 3 [CONTAINS] llap + <-Vertex 17 [CONTAINS] llap + Reduce Output Operator [RS_46] + PartitionCols:_col0, _col1 + Group By Operator [GBY_45] (rows=762 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_41] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_156] (rows=500 width=10) + predicate:value is not null + TableScan [TS_39] (rows=500 width=10) + Output:["key","value"] + <-Vertex 3 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=1100 width=10) @@ -1530,7 +1530,7 @@ Stage-0 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_166] (rows=550 width=10) Conds:MAPJOIN_165._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 10 [BROADCAST_EDGE] llap + <-Vertex 10 [BROADCAST_EDGE] llap BROADCAST [RS_23] PartitionCols:_col0 Select Operator [SEL_18] (rows=500 width=10) @@ -1541,7 +1541,7 @@ Stage-0 default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] <-Map Join Operator [MAPJOIN_165] (rows=288 width=10) Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 9 [BROADCAST_EDGE] llap + <-Vertex 9 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 Select Operator [SEL_15] (rows=25 width=7) @@ -1554,8 +1554,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -1566,7 +1566,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25 width=7) Output:["key","value"] - <-Map 8 [CONTAINS] llap + <-Vertex 8 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -1722,7 +1722,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=266 width=10) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -1748,18 +1748,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_16] Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_25] (rows=266 width=10) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -1775,8 +1775,8 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_0] (rows=242 width=10) default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=242 width=10) Output:["_col1"] @@ -1797,7 +1797,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=266 width=10) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -1823,18 +1823,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_16] Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_25] (rows=266 width=10) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -1850,8 +1850,8 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_0] (rows=242 width=10) default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=242 width=10) Output:["_col1"] @@ -1875,36 +1875,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_26] Group By Operator [GBY_24] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_23] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_23] Group By Operator [GBY_22] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_39] (rows=558 width=10) Conds:Union 2._col0=RS_19._col0(Inner) - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_36] (rows=500 width=10) - predicate:key is not null - TableScan [TS_15] (rows=500 width=10) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_18] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_37] (rows=266 width=10) @@ -1921,7 +1912,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242 width=10) Output:["key"] - <-Map 6 [CONTAINS] llap + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_18] PartitionCols:_col0 Select Operator [SEL_12] (rows=242 width=10) @@ -1930,6 +1921,15 @@ Stage-0 predicate:key is not null TableScan [TS_10] (rows=242 width=10) Output:["key"] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_19] + PartitionCols:_col0 + Select Operator [SEL_17] (rows=500 width=10) + Output:["_col0"] + Filter Operator [FIL_36] (rows=500 width=10) + predicate:key is not null + TableScan [TS_15] (rows=500 width=10) + default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] PREHOOK: query: explain select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value @@ -1946,51 +1946,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 8 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_32] Group By Operator [GBY_30] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_29] + <-Vertex 4 [PARTITION_EDGE] llap + PARTITION [RS_29] Group By Operator [GBY_28] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_51] (rows=587 width=10) Conds:Union 3._col0=RS_25._col0(Inner) - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_47] (rows=500 width=10) - predicate:key is not null - TableScan [TS_21] (rows=500 width=10) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_46] (rows=242 width=10) - predicate:key is not null - TableScan [TS_16] (rows=242 width=10) - Output:["key"] - <-Reducer 2 [CONTAINS] llap + <-Union 3 [SORT_PARTITION_EDGE] + <-Vertex 2 [CONTAINS] llap Reduce Output Operator [RS_24] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_50] (rows=292 width=10) Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_12] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_48] (rows=266 width=10) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -2006,8 +1988,8 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_0] (rows=242 width=10) default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Vertex 7 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=242 width=10) Output:["_col1"] @@ -2015,6 +1997,24 @@ Stage-0 predicate:value is not null TableScan [TS_6] (rows=242 width=10) default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + <-Vertex 8 [CONTAINS] llap + Reduce Output Operator [RS_24] + PartitionCols:_col0 + Select Operator [SEL_18] (rows=242 width=10) + Output:["_col0"] + Filter Operator [FIL_46] (rows=242 width=10) + predicate:key is not null + TableScan [TS_16] (rows=242 width=10) + Output:["key"] + <-Vertex 9 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] + PartitionCols:_col0 + Select Operator [SEL_23] (rows=500 width=10) + Output:["_col0"] + Filter Operator [FIL_47] (rows=500 width=10) + predicate:key is not null + TableScan [TS_21] (rows=500 width=10) + default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] PREHOOK: query: explain SELECT x.key, y.value @@ -2051,22 +2051,28 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 12 <- Union 9 (CONTAINS) - Map 13 <- Union 9 (CONTAINS) - Map 16 <- Map 17 (BROADCAST_EDGE) - Map 18 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 19 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 20 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 21 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) - Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 9 (CONTAINS) + Vertex 13 <- Union 9 (CONTAINS) + Vertex 16 <- Vertex 17 (BROADCAST_EDGE) + Vertex 18 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 19 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 20 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 21 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) + Vertex 8 <- Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2086,7 +2092,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) @@ -2095,7 +2101,45 @@ STAGE PLANS: Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col4 + Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 12 Map Operator Tree: TableScan alias: src @@ -2114,7 +2158,7 @@ STAGE PLANS: Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 13 + Vertex 13 Map Operator Tree: TableScan alias: src @@ -2133,7 +2177,7 @@ STAGE PLANS: Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 14 + Vertex 14 Map Operator Tree: TableScan alias: x @@ -2153,7 +2197,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 15 + Vertex 15 Map Operator Tree: TableScan alias: y @@ -2173,7 +2217,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 16 + Vertex 16 Map Operator Tree: TableScan alias: x @@ -2193,7 +2237,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 17 + 1 Vertex 17 Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) @@ -2221,7 +2265,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: no inputs - Map 17 + Vertex 17 Map Operator Tree: TableScan alias: y @@ -2241,7 +2285,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 18 + Vertex 18 Map Operator Tree: TableScan alias: src1 @@ -2261,7 +2305,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -2276,7 +2320,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 19 + Vertex 19 Map Operator Tree: TableScan alias: src @@ -2296,7 +2340,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -2311,7 +2355,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 20 + Vertex 20 Map Operator Tree: TableScan alias: src @@ -2331,7 +2375,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -2346,7 +2390,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 21 + Vertex 21 Map Operator Tree: TableScan alias: src @@ -2366,7 +2410,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -2381,7 +2425,29 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col4 + Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: src @@ -2401,7 +2467,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) @@ -2410,7 +2476,7 @@ STAGE PLANS: Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x @@ -2436,7 +2502,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: y @@ -2456,7 +2522,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: src1 @@ -2475,72 +2541,6 @@ STAGE PLANS: Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -2577,38 +2577,38 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 23 <- Map 24 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 23 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Union 15 (CONTAINS) +Vertex 16 <- Union 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 13 (CONTAINS) +Vertex 20 <- Union 15 (CONTAINS) +Vertex 23 <- Vertex 24 (BROADCAST_EDGE) +Vertex 25 <- Union 26 (CONTAINS) +Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) +Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (BROADCAST_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 7 (CONTAINS), Vertex 23 (BROADCAST_EDGE) +Vertex 32 <- Union 26 (CONTAINS) +Vertex 33 <- Union 28 (CONTAINS) +Vertex 34 <- Union 30 (CONTAINS) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 llap + Vertex 8 llap File Output Operator [FS_122] Group By Operator [GBY_120] (rows=530 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 31 [CONTAINS] llap + <-Union 7 [SORT_PARTITION_EDGE] + <-Vertex 31 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=1061 width=10) @@ -2617,12 +2617,12 @@ Stage-0 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_167] (rows=484 width=10) Conds:RS_111._col1=SEL_107._col1(Inner),Output:["_col0","_col3"] - <-Map 23 [BROADCAST_EDGE] llap + <-Vertex 23 [BROADCAST_EDGE] llap BROADCAST [RS_111] PartitionCols:_col1 Map Join Operator [MAPJOIN_166] (rows=27 width=7) Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 24 [BROADCAST_EDGE] llap + <-Vertex 24 [BROADCAST_EDGE] llap BROADCAST [RS_109] PartitionCols:_col0 Select Operator [SEL_74] (rows=25 width=7) @@ -2641,19 +2641,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_106] (rows=440 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_100] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=10) - predicate:value is not null - TableScan [TS_98] (rows=500 width=10) - Output:["key","value"] - <-Reducer 29 [CONTAINS] llap + <-Union 30 [SORT_PARTITION_EDGE] + <-Vertex 29 [CONTAINS] llap Reduce Output Operator [RS_105] PartitionCols:_col0, _col1 Group By Operator [GBY_104] (rows=881 width=10) @@ -2662,19 +2651,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_96] (rows=381 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_90] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:value is not null - TableScan [TS_88] (rows=500 width=10) - Output:["key","value"] - <-Reducer 27 [CONTAINS] llap + <-Union 28 [SORT_PARTITION_EDGE] + <-Vertex 27 [CONTAINS] llap Reduce Output Operator [RS_95] PartitionCols:_col0, _col1 Group By Operator [GBY_94] (rows=762 width=10) @@ -2683,8 +2661,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_86] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] llap + <-Union 26 [SORT_PARTITION_EDGE] + <-Vertex 25 [CONTAINS] llap Reduce Output Operator [RS_85] PartitionCols:_col0, _col1 Group By Operator [GBY_84] (rows=525 width=10) @@ -2695,7 +2673,7 @@ Stage-0 predicate:value is not null TableScan [TS_75] (rows=25 width=7) Output:["key","value"] - <-Map 32 [CONTAINS] llap + <-Vertex 32 [CONTAINS] llap Reduce Output Operator [RS_85] PartitionCols:_col0, _col1 Group By Operator [GBY_84] (rows=525 width=10) @@ -2706,15 +2684,37 @@ Stage-0 predicate:value is not null TableScan [TS_78] (rows=500 width=10) Output:["key","value"] - <-Reducer 6 [CONTAINS] llap + <-Vertex 33 [CONTAINS] llap + Reduce Output Operator [RS_95] + PartitionCols:_col0, _col1 + Group By Operator [GBY_94] (rows=762 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_90] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_160] (rows=500 width=10) + predicate:value is not null + TableScan [TS_88] (rows=500 width=10) + Output:["key","value"] + <-Vertex 34 [CONTAINS] llap + Reduce Output Operator [RS_105] + PartitionCols:_col0, _col1 + Group By Operator [GBY_104] (rows=881 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_100] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_161] (rows=500 width=10) + predicate:value is not null + TableScan [TS_98] (rows=500 width=10) + Output:["key","value"] + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=1061 width=10) Output:["_col0","_col1"],keys:_col0, _col1 Group By Operator [GBY_67] (rows=577 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] llap + <-Union 5 [SORT_PARTITION_EDGE] + <-Vertex 18 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=1155 width=10) @@ -2723,49 +2723,20 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_165] (rows=605 width=10) Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] + <-Vertex 17 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_58] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_164] (rows=550 width=10) Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=500 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=500 width=10) - default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] + <-Vertex 16 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_55] PartitionCols:_col1 Select Operator [SEL_48] (rows=381 width=10) Output:["_col1"] Group By Operator [GBY_47] (rows=381 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 15 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 14 [CONTAINS] llap + <-Union 15 [SORT_PARTITION_EDGE] + <-Vertex 14 [CONTAINS] llap Reduce Output Operator [RS_46] PartitionCols:_col0, _col1 Group By Operator [GBY_45] (rows=762 width=10) @@ -2774,8 +2745,8 @@ Stage-0 Output:["_col0","_col1"] Group By Operator [GBY_37] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap + <-Union 13 [SORT_PARTITION_EDGE] + <-Vertex 12 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -2786,7 +2757,7 @@ Stage-0 predicate:value is not null TableScan [TS_26] (rows=25 width=7) Output:["key","value"] - <-Map 19 [CONTAINS] llap + <-Vertex 19 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -2797,7 +2768,36 @@ Stage-0 predicate:value is not null TableScan [TS_29] (rows=500 width=10) Output:["key","value"] - <-Reducer 4 [CONTAINS] llap + <-Vertex 20 [CONTAINS] llap + Reduce Output Operator [RS_46] + PartitionCols:_col0, _col1 + Group By Operator [GBY_45] (rows=762 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_41] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_153] (rows=500 width=10) + predicate:value is not null + TableScan [TS_39] (rows=500 width=10) + Output:["key","value"] + <-Vertex 21 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_56] + PartitionCols:_col1 + Select Operator [SEL_51] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_154] (rows=500 width=10) + predicate:(key is not null and value is not null) + TableScan [TS_49] (rows=500 width=10) + default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Vertex 22 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_59] + PartitionCols:_col0 + Select Operator [SEL_54] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_155] (rows=500 width=10) + predicate:key is not null + TableScan [TS_52] (rows=500 width=10) + default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Vertex 4 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=1155 width=10) @@ -2806,8 +2806,8 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_163] (rows=550 width=10) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Select Operator [SEL_18] (rows=500 width=10) Output:["_col0","_col1"] @@ -2815,12 +2815,12 @@ Stage-0 predicate:key is not null TableScan [TS_16] (rows=500 width=10) default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] PartitionCols:_col2 Map Join Operator [MAPJOIN_162] (rows=288 width=10) Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 10 [BROADCAST_EDGE] llap + <-Vertex 10 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 Select Operator [SEL_15] (rows=25 width=7) @@ -2833,8 +2833,8 @@ Stage-0 Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -2845,7 +2845,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25 width=7) Output:["key","value"] - <-Map 9 [CONTAINS] llap + <-Vertex 9 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -2934,22 +2934,28 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 12 <- Union 9 (CONTAINS) - Map 13 <- Union 9 (CONTAINS) - Map 16 <- Map 17 (BROADCAST_EDGE) - Map 18 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 19 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 20 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 21 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) - Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 9 (CONTAINS) + Vertex 13 <- Union 9 (CONTAINS) + Vertex 16 <- Vertex 17 (BROADCAST_EDGE) + Vertex 18 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 19 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 20 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 21 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) + Vertex 8 <- Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2969,7 +2975,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) @@ -2978,26 +2984,81 @@ STAGE PLANS: Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col4 + Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.a + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.b + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.c + Vertex 12 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 13 + Vertex 13 Map Operator Tree: TableScan alias: src @@ -3016,7 +3077,7 @@ STAGE PLANS: Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 14 + Vertex 14 Map Operator Tree: TableScan alias: x @@ -3036,7 +3097,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 15 + Vertex 15 Map Operator Tree: TableScan alias: y @@ -3056,7 +3117,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 16 + Vertex 16 Map Operator Tree: TableScan alias: x @@ -3076,7 +3137,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 17 + 1 Vertex 17 Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) @@ -3104,7 +3165,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: no inputs - Map 17 + Vertex 17 Map Operator Tree: TableScan alias: y @@ -3124,7 +3185,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 18 + Vertex 18 Map Operator Tree: TableScan alias: src1 @@ -3144,7 +3205,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -3176,7 +3237,7 @@ STAGE PLANS: name: default.c Execution mode: llap LLAP IO: no inputs - Map 19 + Vertex 19 Map Operator Tree: TableScan alias: src @@ -3196,7 +3257,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -3228,7 +3289,7 @@ STAGE PLANS: name: default.c Execution mode: llap LLAP IO: no inputs - Map 20 + Vertex 20 Map Operator Tree: TableScan alias: src @@ -3248,7 +3309,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -3280,7 +3341,7 @@ STAGE PLANS: name: default.c Execution mode: llap LLAP IO: no inputs - Map 21 + Vertex 21 Map Operator Tree: TableScan alias: src @@ -3300,7 +3361,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col3 input vertices: - 0 Map 16 + 0 Vertex 16 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) @@ -3332,7 +3393,46 @@ STAGE PLANS: name: default.c Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col4 + Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.a + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.b + File Output Operator + compressed: false + Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.c + Vertex 5 Map Operator Tree: TableScan alias: src @@ -3352,7 +3452,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) @@ -3361,7 +3461,7 @@ STAGE PLANS: Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x @@ -3387,7 +3487,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: y @@ -3407,7 +3507,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: src1 @@ -3426,106 +3526,6 @@ STAGE PLANS: Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 9 - Vertex: Union 9 Stage: Stage-4 Dependency Collection @@ -3604,27 +3604,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 23 <- Map 24 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 23 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Union 15 (CONTAINS) +Vertex 16 <- Union 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 13 (CONTAINS) +Vertex 20 <- Union 15 (CONTAINS) +Vertex 23 <- Vertex 24 (BROADCAST_EDGE) +Vertex 25 <- Union 26 (CONTAINS) +Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) +Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (BROADCAST_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 7 (CONTAINS), Vertex 23 (BROADCAST_EDGE) +Vertex 32 <- Union 26 (CONTAINS) +Vertex 33 <- Union 28 (CONTAINS) +Vertex 34 <- Union 30 (CONTAINS) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-5 Stats-Aggr Operator @@ -3634,13 +3634,13 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 - Reducer 8 llap + Vertex 8 llap File Output Operator [FS_123] table:{"name:":"default.a"} Group By Operator [GBY_120] (rows=530 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 31 [CONTAINS] llap + <-Union 7 [SORT_PARTITION_EDGE] + <-Vertex 31 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=1061 width=10) @@ -3649,12 +3649,12 @@ Stage-5 Output:["_col0","_col1"] Map Join Operator [MAPJOIN_172] (rows=484 width=10) Conds:RS_111._col1=SEL_107._col1(Inner),Output:["_col0","_col3"] - <-Map 23 [BROADCAST_EDGE] llap + <-Vertex 23 [BROADCAST_EDGE] llap BROADCAST [RS_111] PartitionCols:_col1 Map Join Operator [MAPJOIN_171] (rows=27 width=7) Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 24 [BROADCAST_EDGE] llap + <-Vertex 24 [BROADCAST_EDGE] llap BROADCAST [RS_109] PartitionCols:_col0 Select Operator [SEL_74] (rows=25 width=7) @@ -3673,19 +3673,8 @@ Stage-5 Output:["_col1"] Group By Operator [GBY_106] (rows=440 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_100] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_166] (rows=500 width=10) - predicate:value is not null - TableScan [TS_98] (rows=500 width=10) - Output:["key","value"] - <-Reducer 29 [CONTAINS] llap + <-Union 30 [SORT_PARTITION_EDGE] + <-Vertex 29 [CONTAINS] llap Reduce Output Operator [RS_105] PartitionCols:_col0, _col1 Group By Operator [GBY_104] (rows=881 width=10) @@ -3694,19 +3683,8 @@ Stage-5 Output:["_col0","_col1"] Group By Operator [GBY_96] (rows=381 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_90] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_165] (rows=500 width=10) - predicate:value is not null - TableScan [TS_88] (rows=500 width=10) - Output:["key","value"] - <-Reducer 27 [CONTAINS] llap + <-Union 28 [SORT_PARTITION_EDGE] + <-Vertex 27 [CONTAINS] llap Reduce Output Operator [RS_95] PartitionCols:_col0, _col1 Group By Operator [GBY_94] (rows=762 width=10) @@ -3715,8 +3693,8 @@ Stage-5 Output:["_col0","_col1"] Group By Operator [GBY_86] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] llap + <-Union 26 [SORT_PARTITION_EDGE] + <-Vertex 25 [CONTAINS] llap Reduce Output Operator [RS_85] PartitionCols:_col0, _col1 Group By Operator [GBY_84] (rows=525 width=10) @@ -3727,7 +3705,7 @@ Stage-5 predicate:value is not null TableScan [TS_75] (rows=25 width=7) Output:["key","value"] - <-Map 32 [CONTAINS] llap + <-Vertex 32 [CONTAINS] llap Reduce Output Operator [RS_85] PartitionCols:_col0, _col1 Group By Operator [GBY_84] (rows=525 width=10) @@ -3738,15 +3716,37 @@ Stage-5 predicate:value is not null TableScan [TS_78] (rows=500 width=10) Output:["key","value"] - <-Reducer 6 [CONTAINS] llap + <-Vertex 33 [CONTAINS] llap + Reduce Output Operator [RS_95] + PartitionCols:_col0, _col1 + Group By Operator [GBY_94] (rows=762 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_90] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_165] (rows=500 width=10) + predicate:value is not null + TableScan [TS_88] (rows=500 width=10) + Output:["key","value"] + <-Vertex 34 [CONTAINS] llap + Reduce Output Operator [RS_105] + PartitionCols:_col0, _col1 + Group By Operator [GBY_104] (rows=881 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_100] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_166] (rows=500 width=10) + predicate:value is not null + TableScan [TS_98] (rows=500 width=10) + Output:["key","value"] + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_119] PartitionCols:_col0, _col1 Group By Operator [GBY_118] (rows=1061 width=10) Output:["_col0","_col1"],keys:_col0, _col1 Group By Operator [GBY_67] (rows=577 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] llap + <-Union 5 [SORT_PARTITION_EDGE] + <-Vertex 18 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=1155 width=10) @@ -3755,49 +3755,20 @@ Stage-5 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_170] (rows=605 width=10) Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] + <-Vertex 17 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_58] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_169] (rows=550 width=10) Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=500 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=500 width=10) - default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] + <-Vertex 16 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_55] PartitionCols:_col1 Select Operator [SEL_48] (rows=381 width=10) Output:["_col1"] Group By Operator [GBY_47] (rows=381 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 15 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 14 [CONTAINS] llap + <-Union 15 [SORT_PARTITION_EDGE] + <-Vertex 14 [CONTAINS] llap Reduce Output Operator [RS_46] PartitionCols:_col0, _col1 Group By Operator [GBY_45] (rows=762 width=10) @@ -3806,8 +3777,8 @@ Stage-5 Output:["_col0","_col1"] Group By Operator [GBY_37] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap + <-Union 13 [SORT_PARTITION_EDGE] + <-Vertex 12 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -3818,7 +3789,7 @@ Stage-5 predicate:value is not null TableScan [TS_26] (rows=25 width=7) Output:["key","value"] - <-Map 19 [CONTAINS] llap + <-Vertex 19 [CONTAINS] llap Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=525 width=10) @@ -3829,7 +3800,36 @@ Stage-5 predicate:value is not null TableScan [TS_29] (rows=500 width=10) Output:["key","value"] - <-Reducer 4 [CONTAINS] llap + <-Vertex 20 [CONTAINS] llap + Reduce Output Operator [RS_46] + PartitionCols:_col0, _col1 + Group By Operator [GBY_45] (rows=762 width=10) + Output:["_col0","_col1"],keys:_col1, _col0 + Select Operator [SEL_41] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_158] (rows=500 width=10) + predicate:value is not null + TableScan [TS_39] (rows=500 width=10) + Output:["key","value"] + <-Vertex 21 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_56] + PartitionCols:_col1 + Select Operator [SEL_51] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_159] (rows=500 width=10) + predicate:(key is not null and value is not null) + TableScan [TS_49] (rows=500 width=10) + default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Vertex 22 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_59] + PartitionCols:_col0 + Select Operator [SEL_54] (rows=500 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_160] (rows=500 width=10) + predicate:key is not null + TableScan [TS_52] (rows=500 width=10) + default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + <-Vertex 4 [CONTAINS] llap Reduce Output Operator [RS_66] PartitionCols:_col0, _col1 Group By Operator [GBY_65] (rows=1155 width=10) @@ -3838,8 +3838,8 @@ Stage-5 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_168] (rows=550 width=10) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] + <-Vertex 11 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_23] PartitionCols:_col0 Select Operator [SEL_18] (rows=500 width=10) Output:["_col0","_col1"] @@ -3847,12 +3847,12 @@ Stage-5 predicate:key is not null TableScan [TS_16] (rows=500 width=10) default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] PartitionCols:_col2 Map Join Operator [MAPJOIN_167] (rows=288 width=10) Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 10 [BROADCAST_EDGE] llap + <-Vertex 10 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 Select Operator [SEL_15] (rows=25 width=7) @@ -3865,8 +3865,8 @@ Stage-5 Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap + <-Union 2 [SORT_PARTITION_EDGE] + <-Vertex 1 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -3877,7 +3877,7 @@ Stage-5 predicate:value is not null TableScan [TS_0] (rows=25 width=7) Output:["key","value"] - <-Map 9 [CONTAINS] llap + <-Vertex 9 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Group By Operator [GBY_9] (rows=525 width=10) @@ -3940,10 +3940,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -3953,29 +3953,20 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 5 llap + Vertex 5 llap File Output Operator [FS_21] table:{"name:":"default.dest1"} Group By Operator [GBY_19] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_18] PartitionCols:_col0 Group By Operator [GBY_17] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) Group By Operator [GBY_14] (rows=1 width=272) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=1 width=272) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - <-Reducer 2 [CONTAINS] llap + <-Union 3 [SORT_PARTITION_EDGE] + <-Vertex 2 [CONTAINS] llap Reduce Output Operator [RS_13] PartitionCols:_col0, _col1 Group By Operator [GBY_12] (rows=1 width=272) @@ -3984,13 +3975,22 @@ Stage-4 Output:["_col0","_col1"] Group By Operator [GBY_5] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_4] Group By Operator [GBY_3] (rows=1 width=8) Output:["_col0"],aggregations:["count(1)"] Select Operator [SEL_1] (rows=500 width=10) TableScan [TS_0] (rows=500 width=10) default@src,s1,Tbl:COMPLETE,Col:COMPLETE + <-Vertex 6 [CONTAINS] llap + Reduce Output Operator [RS_13] + PartitionCols:_col0, _col1 + Group By Operator [GBY_12] (rows=1 width=272) + Output:["_col0","_col1"],keys:_col0, _col1 + Select Operator [SEL_8] (rows=500 width=10) + Output:["_col0","_col1"] + TableScan [TS_7] (rows=500 width=10) + Output:["key","value"] File Output Operator [FS_27] table:{"name:":"default.dest2"} Select Operator [SEL_26] (rows=1 width=456) @@ -4012,30 +4012,30 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Unique_join]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Vertex 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=4400 width=10) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_9] (rows=4400 width=10) Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key TableScan [TS_0] (rows=500 width=10) default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_4] PartitionCols:key TableScan [TS_1] (rows=25 width=7) default@src1,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 4 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:key TableScan [TS_2] (rows=2000 width=10) default@srcpart,c,Tbl:COMPLETE,Col:NONE,Output:["key"] @@ -4055,27 +4055,27 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Vertex 2 File Output Operator [FS_9] Transform Operator [SCR_8] (rows=550 width=10) command:cat Merge Join Operator [MERGEJOIN_14] (rows=550 width=10) Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_3] PartitionCols:key Filter Operator [FIL_12] (rows=500 width=10) predicate:key is not null TableScan [TS_0] (rows=500 width=10) default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_5] PartitionCols:key Filter Operator [FIL_13] (rows=500 width=10) predicate:key is not null @@ -4111,11 +4111,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Map 7 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) +Vertex 7 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -4125,67 +4125,67 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_19] table:{"name:":"default.dest1"} Group By Operator [GBY_17] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap + <-Union 3 [SORT_PARTITION_EDGE] + <-Vertex 2 [CONTAINS] llap Reduce Output Operator [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_8] (rows=500 width=10) + Select Operator [SEL_6] (rows=1 width=272) Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] + Group By Operator [GBY_5] (rows=1 width=8) + Output:["_col0"],aggregations:["count(VALUE._col0)"] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_4] + Group By Operator [GBY_3] (rows=1 width=8) + Output:["_col0"],aggregations:["count(1)"] + Select Operator [SEL_1] (rows=500 width=10) + TableScan [TS_0] (rows=500 width=10) + default@src,s1,Tbl:COMPLETE,Col:COMPLETE Reduce Output Operator [RS_22] PartitionCols:_col0, _col1 Group By Operator [GBY_21] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_8] - <-Map 7 [CONTAINS] llap + Please refer to the previous Select Operator [SEL_6] + <-Vertex 6 [CONTAINS] llap Reduce Output Operator [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_12] (rows=500 width=10) + Select Operator [SEL_8] (rows=500 width=10) Output:["_col0","_col1"] - TableScan [TS_11] (rows=500 width=10) + TableScan [TS_7] (rows=500 width=10) Output:["key","value"] Reduce Output Operator [RS_22] PartitionCols:_col0, _col1 Group By Operator [GBY_21] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_12] - <-Reducer 2 [CONTAINS] llap + Please refer to the previous Select Operator [SEL_8] + <-Vertex 7 [CONTAINS] llap Reduce Output Operator [RS_16] PartitionCols:_col0 Group By Operator [GBY_15] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_6] (rows=1 width=272) + Select Operator [SEL_12] (rows=500 width=10) Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE + TableScan [TS_11] (rows=500 width=10) + Output:["key","value"] Reduce Output Operator [RS_22] PartitionCols:_col0, _col1 Group By Operator [GBY_21] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_6] - Reducer 5 llap + Please refer to the previous Select Operator [SEL_12] + Vertex 5 llap File Output Operator [FS_25] table:{"name:":"default.dest2"} Group By Operator [GBY_23] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + <- Please refer to the previous Union 3 [SORT_PARTITION_EDGE] Stage-5 Stats-Aggr Operator Stage-1 @@ -4216,10 +4216,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -4229,27 +4229,13 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_16] table:{"name:":"default.dest1"} Group By Operator [GBY_14] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_19] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_8] - <-Reducer 2 [CONTAINS] llap + <-Union 3 [SORT_PARTITION_EDGE] + <-Vertex 2 [CONTAINS] llap Reduce Output Operator [RS_13] PartitionCols:_col0 Group By Operator [GBY_12] (rows=1 width=280) @@ -4258,8 +4244,8 @@ Stage-4 Output:["_col0","_col1"] Group By Operator [GBY_5] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_4] Group By Operator [GBY_3] (rows=1 width=8) Output:["_col0"],aggregations:["count(1)"] Select Operator [SEL_1] (rows=500 width=10) @@ -4270,12 +4256,26 @@ Stage-4 Group By Operator [GBY_18] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) Please refer to the previous Select Operator [SEL_6] - Reducer 5 llap + <-Vertex 6 [CONTAINS] llap + Reduce Output Operator [RS_13] + PartitionCols:_col0 + Group By Operator [GBY_12] (rows=1 width=280) + Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) + Select Operator [SEL_8] (rows=500 width=10) + Output:["_col0","_col1"] + TableScan [TS_7] (rows=500 width=10) + Output:["key","value"] + Reduce Output Operator [RS_19] + PartitionCols:_col0, _col1 + Group By Operator [GBY_18] (rows=1 width=464) + Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) + Please refer to the previous Select Operator [SEL_8] + Vertex 5 llap File Output Operator [FS_22] table:{"name:":"default.dest2"} Group By Operator [GBY_20] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + <- Please refer to the previous Union 3 [SORT_PARTITION_EDGE] Stage-5 Stats-Aggr Operator Stage-1 diff --git ql/src/test/results/clientpositive/llap/explainuser_4.q.out ql/src/test/results/clientpositive/llap/explainuser_4.q.out index 9804569..7a1447a 100644 --- ql/src/test/results/clientpositive/llap/explainuser_4.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_4.q.out @@ -17,39 +17,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=2166 width=620) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_17] (rows=2166 width=620) - Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365 width=251) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=251) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1019 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1019 width=251) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=251) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -96,41 +71,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=2166 width=8) - Conds:RS_6._col0=RS_7._col0(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365 width=2) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365 width=2) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=2) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["cint"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019 width=8) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1019 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 3 PREHOOK: query: select count(*) @@ -170,47 +118,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=615 width=12) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_12] (rows=615 width=12) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=615 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_21] (rows=2166 width=4) - Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365 width=5) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=1365 width=5) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=5) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019 width=8) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1019 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -256,39 +172,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1501 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_17] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -335,41 +226,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_19] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 3 PREHOOK: query: select count(*) @@ -409,47 +273,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=750 width=215) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_12] (rows=750 width=215) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=1501 width=215) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Map Join Operator [MAPJOIN_21] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -476,7 +308,7 @@ POSTHOOK: Input: default@alltypesorc -3799 1 10782 1 NULL 6 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from alltypesorc a left outer join alltypesorc b @@ -490,26 +322,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Merge Join Operator [MERGEJOIN_9] (rows=150994944 width=431) - Conds:(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],residual filter predicates:{((_col2 = _col14) or _col1 BETWEEN 1 AND 10)} - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Select Operator [SEL_1] (rows=12288 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] - Select Operator [SEL_3] (rows=12288 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - TableScan [TS_2] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out index c5cdb77..2bea0ef 100644 --- ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out +++ ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: f @@ -117,7 +117,68 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [f] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col3 + Position of Big Table: 0 + Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col0 (type: int) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col5 + Position of Big Table: 0 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: m @@ -194,7 +255,7 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [m] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: g @@ -270,67 +331,6 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [g] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col3 - Position of Big Table: 0 - Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col0 (type: int) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col5 - Position of Big Table: 0 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/groupby1.q.out ql/src/test/results/clientpositive/llap/groupby1.q.out index 0eecbb6..88c16c3 100644 --- ql/src/test/results/clientpositive/llap/groupby1.q.out +++ ql/src/test/results/clientpositive/llap/groupby1.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -59,7 +59,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/groupby2.q.out ql/src/test/results/clientpositive/llap/groupby2.q.out index 29b85d1..d3ceed8 100644 --- ql/src/test/results/clientpositive/llap/groupby2.q.out +++ ql/src/test/results/clientpositive/llap/groupby2.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,7 +45,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -60,7 +60,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/groupby3.q.out ql/src/test/results/clientpositive/llap/groupby3.q.out index 3495de6..4a36689 100644 --- ql/src/test/results/clientpositive/llap/groupby3.q.out +++ ql/src/test/results/clientpositive/llap/groupby3.q.out @@ -43,11 +43,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -63,7 +63,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -75,7 +75,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double), _col1 (type: struct), _col2 (type: struct), _col3 (type: string), _col4 (type: string), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/groupby_resolution.q.out ql/src/test/results/clientpositive/llap/groupby_resolution.q.out index f2a6ab0..4436ba1 100644 --- ql/src/test/results/clientpositive/llap/groupby_resolution.q.out +++ ql/src/test/results/clientpositive/llap/groupby_resolution.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -30,7 +30,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -66,10 +66,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -85,7 +85,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,11 +121,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -141,7 +141,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -156,7 +156,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -192,11 +192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -212,7 +212,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -227,7 +227,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -263,10 +263,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -289,7 +289,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,10 +325,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -351,7 +351,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -414,7 +414,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -429,7 +429,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -465,11 +465,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -492,7 +492,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -507,7 +507,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -638,12 +638,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -665,7 +665,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -680,7 +680,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -695,7 +695,7 @@ STAGE PLANS: Map-reduce partition columns: 0 (type: int) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -785,10 +785,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -805,7 +805,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/having.q.out ql/src/test/results/clientpositive/llap/having.q.out index 267254c..7353c9a 100644 --- ql/src/test/results/clientpositive/llap/having.q.out +++ ql/src/test/results/clientpositive/llap/having.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -37,7 +37,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -102,10 +102,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -127,7 +127,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,10 +479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -505,7 +505,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -755,10 +755,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -780,7 +780,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -956,10 +956,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -982,7 +982,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1228,10 +1228,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1254,7 +1254,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index 33d88ce..8d6051d 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -32,11 +32,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -68,7 +68,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -87,21 +102,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -153,11 +153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -176,7 +176,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -189,7 +189,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -208,21 +223,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -272,11 +272,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -295,7 +295,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -308,7 +308,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -327,21 +342,6 @@ STAGE PLANS: Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -389,11 +389,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -412,7 +412,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -425,7 +425,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -444,21 +459,6 @@ STAGE PLANS: Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -506,11 +506,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -526,7 +526,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -539,7 +539,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -555,21 +570,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -617,11 +617,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -637,7 +637,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -650,7 +650,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -666,21 +681,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -760,11 +760,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -783,7 +783,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -796,7 +796,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -815,21 +830,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -881,11 +881,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -904,7 +904,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -918,7 +918,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -937,21 +952,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1003,11 +1003,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1026,7 +1026,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1039,7 +1039,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1058,21 +1073,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1124,11 +1124,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1147,7 +1147,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -1161,7 +1161,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1180,21 +1195,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1253,7 +1253,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1273,10 +1273,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1296,7 +1296,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1311,7 +1311,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1336,7 +1336,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1451,7 +1451,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1471,10 +1471,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1494,7 +1494,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1509,7 +1509,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1534,7 +1534,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out index ee9930f..05adcbb 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out @@ -35,11 +35,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -54,7 +54,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -71,8 +71,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -85,22 +85,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -115,6 +100,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -166,11 +166,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -185,7 +185,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -202,8 +202,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -216,22 +216,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -246,6 +231,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -308,11 +308,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -327,7 +327,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -346,9 +346,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -361,7 +361,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -376,7 +391,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -391,21 +406,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -461,11 +461,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -480,7 +480,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -499,9 +499,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -514,7 +514,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -529,7 +544,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -544,21 +559,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -629,14 +629,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -651,7 +653,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -666,7 +668,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -683,8 +685,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -697,7 +699,40 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -712,7 +747,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -727,7 +762,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -744,8 +779,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -758,40 +793,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -809,8 +811,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -879,14 +879,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -901,7 +903,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -916,7 +918,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -933,8 +935,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -947,7 +949,40 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -962,7 +997,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -977,7 +1012,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -994,8 +1029,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1008,40 +1043,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1059,8 +1061,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -1140,11 +1140,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1160,7 +1160,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1178,8 +1178,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1190,8 +1190,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1204,7 +1204,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1219,7 +1234,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1234,7 +1249,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1249,21 +1264,6 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1335,11 +1335,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1355,7 +1355,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1373,8 +1373,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1385,8 +1385,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1399,7 +1399,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1414,7 +1429,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1429,7 +1444,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1444,21 +1459,6 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out index 3375fa8..af6f03e 100644 --- ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out +++ ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,23 +45,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '105') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: '105' (type: string) - sort order: + - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -73,7 +57,7 @@ STAGE PLANS: 0 '105' (type: string) 1 '105' (type: string) input vertices: - 1 Reducer 4 + 1 Vertex 4 Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '105' (type: string), 'val_105' (type: string) @@ -86,7 +70,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '105') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: '105' (type: string) + sort order: + + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/insert1.q.out ql/src/test/results/clientpositive/llap/insert1.q.out index aa09585..3ff7427 100644 --- ql/src/test/results/clientpositive/llap/insert1.q.out +++ ql/src/test/results/clientpositive/llap/insert1.q.out @@ -39,7 +39,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -93,7 +93,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -161,7 +161,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -215,7 +215,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -277,7 +277,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: insert2 diff --git ql/src/test/results/clientpositive/llap/insert_into1.q.out ql/src/test/results/clientpositive/llap/insert_into1.q.out index 5caefe8..f0a18e3 100644 --- ql/src/test/results/clientpositive/llap/insert_into1.q.out +++ ql/src/test/results/clientpositive/llap/insert_into1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,7 +45,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -147,10 +147,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -167,7 +167,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -269,10 +269,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -289,7 +289,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -391,7 +391,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: _dummy_table @@ -453,7 +453,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: _dummy_table @@ -526,7 +526,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: values__tmp__table__1 @@ -582,10 +582,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: insert_into1 @@ -603,7 +603,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/insert_into2.q.out ql/src/test/results/clientpositive/llap/insert_into2.q.out index a42c651..ddedb60 100644 --- ql/src/test/results/clientpositive/llap/insert_into2.q.out +++ ql/src/test/results/clientpositive/llap/insert_into2.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -49,7 +49,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -192,10 +192,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -212,7 +212,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -324,10 +324,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -344,7 +344,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -462,10 +462,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: insert_into2 @@ -483,7 +483,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/intersect_all.q.out ql/src/test/results/clientpositive/llap/intersect_all.q.out index 4dac8a2..fa18977 100644 --- ql/src/test/results/clientpositive/llap/intersect_all.q.out +++ ql/src/test/results/clientpositive/llap/intersect_all.q.out @@ -146,35 +146,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -197,7 +176,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -218,7 +197,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -248,7 +227,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -269,8 +271,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -799,83 +799,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -898,7 +831,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -919,7 +852,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -940,7 +873,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -970,7 +903,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -991,7 +947,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1012,8 +991,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1542,12 +1542,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1568,28 +1570,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1615,7 +1596,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1645,7 +1626,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1671,8 +1673,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/intersect_distinct.q.out ql/src/test/results/clientpositive/llap/intersect_distinct.q.out index 3698f03..6ac445f 100644 --- ql/src/test/results/clientpositive/llap/intersect_distinct.q.out +++ ql/src/test/results/clientpositive/llap/intersect_distinct.q.out @@ -144,35 +144,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -195,7 +174,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -216,7 +195,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -239,7 +218,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,8 +262,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -599,83 +599,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -698,7 +631,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -719,7 +652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -740,7 +673,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -763,7 +696,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -784,7 +740,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,8 +784,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1144,12 +1144,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1170,28 +1172,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1217,7 +1198,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1240,7 +1221,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1266,8 +1268,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/intersect_merge.q.out ql/src/test/results/clientpositive/llap/intersect_merge.q.out index 6217785..28658c0 100644 --- ql/src/test/results/clientpositive/llap/intersect_merge.q.out +++ ql/src/test/results/clientpositive/llap/intersect_merge.q.out @@ -43,38 +43,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 11 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -97,30 +76,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 11 Map Operator Tree: TableScan alias: b @@ -143,51 +120,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -208,7 +141,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -229,7 +162,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -252,28 +185,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -281,7 +216,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -294,36 +229,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) -PREHOOK: type: QUERY -POSTHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -346,7 +252,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -369,7 +296,36 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) +PREHOOK: type: QUERY +POSTHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -392,30 +348,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -436,7 +369,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -457,7 +390,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -480,28 +413,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -509,7 +444,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -522,37 +457,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -575,30 +480,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 8 Execution mode: llap - LLAP IO: no inputs - Map 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -621,7 +524,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -644,30 +577,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -688,7 +598,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -709,7 +642,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -730,7 +663,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -753,28 +686,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -782,7 +717,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -795,37 +730,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b -PREHOOK: type: QUERY -POSTHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -848,30 +753,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 8 Execution mode: llap - LLAP IO: no inputs - Map 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -894,7 +797,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b +PREHOOK: type: QUERY +POSTHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -917,30 +850,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -961,72 +871,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col2 = 5) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1034,7 +902,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -1047,7 +915,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1068,60 +936,74 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 = 5) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 7 Map Operator Tree: TableScan alias: b @@ -1144,7 +1026,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -1167,7 +1070,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1190,30 +1123,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1234,7 +1144,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1255,7 +1188,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1276,7 +1209,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1299,28 +1232,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1328,7 +1263,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -1341,35 +1276,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -1392,7 +1299,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -1415,7 +1343,35 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1438,7 +1394,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1459,7 +1415,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1482,7 +1438,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1503,7 +1482,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1524,8 +1526,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1546,59 +1546,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1621,7 +1577,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1642,7 +1598,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1672,7 +1628,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1693,7 +1672,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1714,8 +1716,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1736,60 +1736,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 3 (CONTAINS), Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 7 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1812,7 +1770,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1833,7 +1791,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1854,7 +1812,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1884,7 +1842,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1905,7 +1886,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1939,10 +1920,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/join0.q.out ql/src/test/results/clientpositive/llap/join0.q.out index 1b23c3d..8238d94 100644 --- ql/src/test/results/clientpositive/llap/join0.q.out +++ ql/src/test/results/clientpositive/llap/join0.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM @@ -24,11 +24,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -46,25 +46,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -79,7 +61,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -93,6 +75,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -100,7 +100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN FORMATTED SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM @@ -118,7 +118,7 @@ SELECT src1.key as k1, src1.value as v1, SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 diff --git ql/src/test/results/clientpositive/llap/join1.q.out ql/src/test/results/clientpositive/llap/join1.q.out index d79a405..b4225ce 100644 --- ql/src/test/results/clientpositive/llap/join1.q.out +++ ql/src/test/results/clientpositive/llap/join1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -47,27 +47,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -90,6 +70,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/join32_lessSize.q.out ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index 5c2e3d4..42dedd1 100644 --- ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -58,13 +58,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 25 + Estimated key counts: Vertex 3 => 25 keys: 0 _col0 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 58 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -131,7 +131,56 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 1 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: x @@ -207,7 +256,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -283,55 +332,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 - Position of Big Table: 1 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -509,12 +509,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -531,13 +531,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 4 => 25 + Estimated key counts: Vertex 4 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Position of Big Table: 0 Statistics: Num rows: 44 Data size: 11616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -602,7 +602,78 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3 + Position of Big Table: 1 + Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: _col3 (type: string) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: z @@ -678,7 +749,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [z] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: w @@ -753,7 +824,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [w] - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y @@ -829,77 +900,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - Position of Big Table: 1 - Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: _col3 (type: string) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 - Position of Big Table: 0 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -1073,11 +1073,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -1154,31 +1154,80 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - Estimated key counts: Map 4 => 25 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - input vertices: - 1 Map 4 - Position of Big Table: 0 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Position of Big Table: 0 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j2 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest_j2 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + Estimated key counts: Vertex 4 => 25 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + input vertices: + 1 Vertex 4 + Position of Big Table: 0 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -1246,7 +1295,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -1322,55 +1371,6 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 0 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j2 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest_j2 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -1546,11 +1546,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -1627,7 +1627,56 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Position of Big Table: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j2 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j2 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1698,7 +1747,33 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: _col0 (type: string) + auto parallelism: true + Vertex 5 Map Operator Tree: TableScan alias: x @@ -1774,81 +1849,6 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j2 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j2 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: _col0 (type: string) - auto parallelism: true Stage: Stage-2 Dependency Collection @@ -2036,11 +2036,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2059,7 +2059,30 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + Vertex 3 Map Operator Tree: TableScan alias: y @@ -2079,7 +2102,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -2093,7 +2116,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2113,29 +2136,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 Stage: Stage-2 Dependency Collection @@ -2292,11 +2292,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -2315,7 +2315,30 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + Vertex 3 Map Operator Tree: TableScan alias: y @@ -2335,7 +2358,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -2349,7 +2372,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2369,29 +2392,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/join46.q.out ql/src/test/results/clientpositive/llap/join46.q.out index 56f6862..d21b449 100644 --- ql/src/test/results/clientpositive/llap/join46.q.out +++ ql/src/test/results/clientpositive/llap/join46.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,24 +75,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -110,6 +93,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -162,10 +162,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -182,27 +182,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -223,6 +203,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -254,7 +254,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 99 2 Mat NULL NULL NULL 101 2 Car 102 2 Del -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -276,10 +276,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -294,25 +294,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -333,6 +315,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -340,7 +340,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -382,10 +382,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -402,24 +402,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -437,6 +420,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -464,7 +464,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -484,10 +484,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -502,22 +502,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -538,6 +523,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -545,7 +545,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -572,7 +572,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -614,22 +614,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -648,6 +633,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -655,7 +655,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -687,7 +687,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -709,10 +709,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -727,22 +727,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -761,6 +746,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -768,7 +768,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -798,7 +798,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -820,10 +820,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -838,22 +838,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -872,6 +857,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -879,7 +879,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -927,10 +927,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -947,24 +947,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -983,6 +966,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1015,7 +1015,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat 102 2 Del 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT col_1, col_2 FROM test1 LEFT OUTER JOIN test2 @@ -1037,10 +1037,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1055,22 +1055,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1093,6 +1078,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1100,7 +1100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT col_1, col_2 FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -1125,7 +1125,7 @@ Mat Ema Bob NULL Car Del Car Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1149,10 +1149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1167,22 +1167,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1201,6 +1186,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1208,7 +1208,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1240,7 +1240,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1262,10 +1262,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1280,22 +1280,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1314,6 +1299,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1321,7 +1321,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1348,7 +1348,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1370,10 +1370,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1388,22 +1388,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1422,6 +1407,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1429,7 +1429,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1479,10 +1479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1499,24 +1499,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1535,6 +1518,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1565,7 +1565,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1589,10 +1589,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1607,22 +1607,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1641,6 +1626,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1648,7 +1648,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1680,7 +1680,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1702,10 +1702,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1720,22 +1720,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1754,6 +1739,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1761,7 +1761,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1791,7 +1791,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1813,10 +1813,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1831,22 +1831,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1865,6 +1850,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1872,7 +1872,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1922,10 +1922,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1942,24 +1942,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1978,6 +1961,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2012,7 +2012,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2062,46 +2062,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -2118,24 +2084,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2151,7 +2100,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2170,7 +2119,41 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 4 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2186,6 +2169,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2193,7 +2193,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1, diff --git ql/src/test/results/clientpositive/llap/join_emit_interval.q.out ql/src/test/results/clientpositive/llap/join_emit_interval.q.out index 7fda074..eace277 100644 --- ql/src/test/results/clientpositive/llap/join_emit_interval.q.out +++ ql/src/test/results/clientpositive/llap/join_emit_interval.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,24 +75,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -113,6 +96,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -141,7 +141,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -161,10 +161,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -179,22 +179,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -215,6 +200,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -222,7 +222,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) diff --git ql/src/test/results/clientpositive/llap/join_filters.q.out ql/src/test/results/clientpositive/llap/join_filters.q.out index b730d30..f0ab223 100644 --- ql/src/test/results/clientpositive/llap/join_filters.q.out +++ ql/src/test/results/clientpositive/llap/join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -25,7 +25,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -39,7 +39,7 @@ POSTHOOK: Input: default@myinput1 48 NULL NULL NULL NULL 40 NULL NULL M3MWtBJdRXSWIJY5Qr/otw== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -53,7 +53,7 @@ NULL NULL 12 35 NULL NULL 48 NULL NULL NULL NULL 40 LNZKrcVNAvaeDALnsg72bw== -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -356,7 +356,7 @@ NULL NULL NULL NULL 12 35 NULL NULL NULL NULL 48 NULL NULL NULL NULL NULL NULL 40 AzUxen/yR7DlsL00zfSITA== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -407,7 +407,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -770,7 +770,7 @@ NULL NULL 48 NULL NULL NULL NULL 135 NULL NULL NULL 35 UBr9lyqgsjDFvooMgQlZ9w== -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -781,7 +781,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -795,7 +795,7 @@ POSTHOOK: Input: default@myinput1 48 NULL NULL NULL NULL 40 NULL NULL M3MWtBJdRXSWIJY5Qr/otw== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -809,7 +809,7 @@ NULL NULL 12 35 NULL NULL 48 NULL NULL NULL NULL 40 LNZKrcVNAvaeDALnsg72bw== -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -1112,7 +1112,7 @@ NULL NULL NULL NULL 12 35 NULL NULL NULL NULL 48 NULL NULL NULL NULL NULL NULL 40 AzUxen/yR7DlsL00zfSITA== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -1163,7 +1163,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/join_nulls.q.out ql/src/test/results/clientpositive/llap/join_nulls.q.out index c9876d3..3e11fe4 100644 --- ql/src/test/results/clientpositive/llap/join_nulls.q.out +++ ql/src/test/results/clientpositive/llap/join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -32,7 +32,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -50,7 +50,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -277,7 +277,7 @@ POSTHOOK: Input: default@myinput1 100 100 100 100 100 100 NULL 35 NULL 35 NULL 35 NULL NULL NULL NULL 48 NULL -Warning: Shuffle Join MERGEJOIN[7][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[7][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/join_nullsafe.q.out ql/src/test/results/clientpositive/llap/join_nullsafe.q.out index 71685aa..b1ddb6b 100644 --- ql/src/test/results/clientpositive/llap/join_nullsafe.q.out +++ ql/src/test/results/clientpositive/llap/join_nullsafe.q.out @@ -27,10 +27,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -43,20 +43,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -79,6 +66,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -118,10 +118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -137,7 +137,31 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int) + 1 value (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -153,7 +177,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -169,30 +193,6 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int) - 1 value (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -223,10 +223,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -239,33 +239,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -290,6 +264,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -347,10 +347,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -365,7 +365,32 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int), value (type: int) + 1 value (type: int), key (type: int) + 2 key (type: int), value (type: int) + nullSafes: [true, false] + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -380,7 +405,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -395,31 +420,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int), value (type: int) - 1 value (type: int), key (type: int) - 2 key (type: int), value (type: int) - nullSafes: [true, false] - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -450,10 +450,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -465,31 +465,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int), key (type: int) - sort order: ++ - Map-reduce partition columns: value (type: int), key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int), value (type: int) - sort order: ++ - Map-reduce partition columns: key (type: int), value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -514,6 +490,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int), key (type: int) + sort order: ++ + Map-reduce partition columns: value (type: int), key (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int), value (type: int) + sort order: ++ + Map-reduce partition columns: key (type: int), value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1589,10 +1589,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1608,23 +1608,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is null (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: null (type: int) - sort order: + - Map-reduce partition columns: null (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1647,6 +1631,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: null (type: int) + sort order: + + Map-reduce partition columns: null (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/lateral_view.q.out ql/src/test/results/clientpositive/llap/lateral_view.q.out index 5bde889..5d5c755 100644 --- ql/src/test/results/clientpositive/llap/lateral_view.q.out +++ ql/src/test/results/clientpositive/llap/lateral_view.q.out @@ -28,11 +28,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -78,7 +78,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -94,7 +94,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -622,10 +622,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tmp_pyang_src_rcfile @@ -646,7 +646,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 184 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index 61b5c12..7441a9c 100644 --- ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -39,24 +39,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -77,6 +60,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -120,10 +120,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -140,24 +140,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -178,6 +161,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -221,11 +221,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -244,24 +244,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -277,7 +260,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -298,6 +281,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -347,12 +347,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -371,41 +371,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,7 +387,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -438,7 +404,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -463,6 +429,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -518,13 +518,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 7 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -541,43 +541,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -598,7 +562,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -614,7 +597,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -633,7 +616,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -649,6 +632,23 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -706,14 +706,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -730,40 +730,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -780,7 +747,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -797,7 +764,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -818,7 +801,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -834,7 +817,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -949,10 +949,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -969,24 +969,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1008,6 +991,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1043,38 +1043,21 @@ on src1.key = src2.key limit 1 offset 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: src2 + alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -1088,7 +1071,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1110,6 +1093,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1153,11 +1153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1177,24 +1177,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1211,7 +1194,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1233,6 +1216,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1281,12 +1281,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -1306,41 +1306,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1357,7 +1323,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1374,7 +1340,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1400,6 +1366,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1454,13 +1454,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 7 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1477,44 +1477,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Offset of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1536,7 +1499,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Offset of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1553,7 +1536,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1573,7 +1556,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1590,6 +1573,23 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1646,14 +1646,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1670,40 +1670,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1720,7 +1687,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1738,7 +1705,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1759,7 +1742,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1776,7 +1759,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/limit_pushdown.q.out ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index e9cf56d..c1a8f4d 100644 --- ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -120,7 +120,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -187,10 +187,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -214,7 +214,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -283,10 +283,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -310,7 +310,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -379,10 +379,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -404,7 +404,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -472,10 +472,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -496,7 +496,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -570,10 +570,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -594,7 +594,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -668,10 +668,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -694,7 +694,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,11 +787,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -814,7 +814,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -829,7 +829,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -904,13 +904,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -934,31 +934,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.3 - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -979,7 +955,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1000,7 +976,31 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1017,7 +1017,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1058,10 +1058,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1079,7 +1079,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1148,10 +1148,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1168,7 +1168,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1315,11 +1315,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1336,7 +1336,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1354,7 +1354,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out index 735ba67..cf7ece0 100644 --- ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out +++ ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -120,7 +120,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -187,11 +187,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -215,7 +215,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -230,7 +230,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -297,11 +297,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -325,7 +325,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -340,7 +340,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -407,11 +407,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -433,7 +433,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -446,7 +446,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 4265 Data size: 25480 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -513,11 +513,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -538,7 +538,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -558,7 +558,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1048 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -625,11 +625,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -650,7 +650,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -670,7 +670,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1048 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -737,11 +737,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -764,7 +764,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -779,7 +779,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1808 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -870,11 +870,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -897,7 +897,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -912,7 +912,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -979,11 +979,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1001,7 +1001,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1016,7 +1016,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1083,10 +1083,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1103,7 +1103,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1250,11 +1250,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1271,7 +1271,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1289,7 +1289,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/lineage3.q.out ql/src/test/results/clientpositive/llap/lineage3.q.out index f092967..430d369 100644 --- ql/src/test/results/clientpositive/llap/lineage3.q.out +++ ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -180,7 +180,7 @@ PREHOOK: Input: default@src1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select key, value from src1 where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY @@ -370,7 +370,7 @@ PREHOOK: query: create table src_dp1 (f string, w string, m int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_dp1 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: from src_dp, src_dp1 insert into dest_dp1 partition (year) select first, word, year insert into dest_dp2 partition (y, m) select first, word, year, month diff --git ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index cdb688b..3f4ba09 100644 --- ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/llap_nullscan.q.out ql/src/test/results/clientpositive/llap/llap_nullscan.q.out index b798e82..2c8e34c 100644 --- ql/src/test/results/clientpositive/llap/llap_nullscan.q.out +++ ql/src/test/results/clientpositive/llap/llap_nullscan.q.out @@ -39,7 +39,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -157,11 +157,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -180,25 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src_orc - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -216,7 +198,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -261,11 +261,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -284,7 +286,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src_orc @@ -304,22 +321,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -334,8 +336,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -368,10 +368,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -387,23 +387,7 @@ STAGE PLANS: value expressions: key (type: string), ds (type: string), hr (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string), ds (type: string), hr (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -425,6 +409,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), ds (type: string), hr (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/llap_partitioned.q.out ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index d35501e..2afd8fe 100644 --- ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -1623,10 +1623,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: oft @@ -1639,7 +1639,7 @@ STAGE PLANS: 1 ctinyint (type: tinyint) outputColumnNames: _col1, _col6, _col7, _col10 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 960 Data size: 497280 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col10 (type: tinyint), _col1 (type: int), _col6 (type: char(255)), _col7 (type: varchar(255)) @@ -1654,7 +1654,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: od @@ -1678,7 +1678,7 @@ STAGE PLANS: Target Input: oft Partition key expr: ctinyint Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: vectorized, llap LLAP IO: all inputs diff --git ql/src/test/results/clientpositive/llap/llap_stats.q.out ql/src/test/results/clientpositive/llap/llap_stats.q.out index f6921f1..b8fc80e 100644 --- ql/src/test/results/clientpositive/llap/llap_stats.q.out +++ ql/src/test/results/clientpositive/llap/llap_stats.q.out @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: llap_stats @@ -126,7 +126,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/llap_udf.q.out ql/src/test/results/clientpositive/llap/llap_udf.q.out index 7a95c8b..acf7038 100644 --- ql/src/test/results/clientpositive/llap/llap_udf.q.out +++ ql/src/test/results/clientpositive/llap/llap_udf.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -95,7 +95,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -135,7 +135,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -181,7 +181,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -229,7 +229,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -267,7 +267,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc diff --git ql/src/test/results/clientpositive/llap/llapdecider.q.out ql/src/test/results/clientpositive/llap/llapdecider.q.out index d514f42..b25e5e2 100644 --- ql/src/test/results/clientpositive/llap/llapdecider.q.out +++ ql/src/test/results/clientpositive/llap/llapdecider.q.out @@ -11,11 +11,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -36,7 +36,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -49,7 +49,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -94,11 +94,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -119,7 +119,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -132,7 +132,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -165,11 +165,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -192,7 +192,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -206,7 +206,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -248,11 +248,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -275,7 +275,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -289,7 +289,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -323,11 +323,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -347,25 +347,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -380,7 +362,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -393,6 +375,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -437,27 +437,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -473,7 +453,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -487,6 +467,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -507,11 +507,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -531,25 +531,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -565,7 +547,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -579,6 +561,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -599,11 +599,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -621,25 +621,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -654,7 +636,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -667,6 +649,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -687,11 +687,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -709,25 +709,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -742,7 +724,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -755,6 +737,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -775,11 +775,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -799,27 +799,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -834,7 +814,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -847,6 +827,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -867,11 +867,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -889,25 +889,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -922,7 +904,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -935,6 +917,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -955,11 +955,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -979,27 +979,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1015,7 +995,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1029,6 +1009,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1055,10 +1055,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1081,7 +1081,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1116,10 +1116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1140,7 +1140,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1174,10 +1174,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1198,7 +1198,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1232,10 +1232,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1258,7 +1258,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out index 6ad9af8..7fdf45a 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out index ce5517a..14e5064 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out @@ -45,10 +45,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -64,7 +64,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out index b34975f..b035547 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out @@ -51,18 +51,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index 9c4e8a8..13af9d1 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -38,7 +38,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out index c7c4c70..6856c0a 100644 --- ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out @@ -107,10 +107,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: expod1 @@ -139,7 +139,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 85 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col2) (type: boolean) @@ -177,7 +177,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 85 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col2) (type: boolean) @@ -195,7 +195,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: expod2 diff --git ql/src/test/results/clientpositive/llap/mapjoin2.q.out ql/src/test/results/clientpositive/llap/mapjoin2.q.out index cdee70b..3a74b84 100644 --- ql/src/test/results/clientpositive/llap/mapjoin2.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin2.q.out @@ -22,7 +22,7 @@ POSTHOOK: type: QUERY POSTHOOK: Output: default@tbl POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -32,7 +32,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### 1 one true true -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -42,7 +42,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### true true 2 two -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl diff --git ql/src/test/results/clientpositive/llap/mapjoin3.q.out ql/src/test/results/clientpositive/llap/mapjoin3.q.out index cf1bebb..1aa212e 100644 --- ql/src/test/results/clientpositive/llap/mapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin3.q.out @@ -98,10 +98,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -118,7 +118,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col2 (type: varchar(100)), _col3 (type: varchar(100)) @@ -133,7 +133,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_1 diff --git ql/src/test/results/clientpositive/llap/mapjoin46.q.out ql/src/test/results/clientpositive/llap/mapjoin46.q.out index 73960ce..cc9893c 100644 --- ql/src/test/results/clientpositive/llap/mapjoin46.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin46.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,7 +75,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -86,7 +86,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -155,10 +155,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -178,7 +178,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -189,7 +189,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -240,7 +240,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -262,10 +262,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -285,7 +285,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 116 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -296,7 +296,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -321,7 +321,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -363,10 +363,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -383,7 +383,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -400,7 +400,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -438,7 +438,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -458,10 +458,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -481,7 +481,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -492,7 +492,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -514,7 +514,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -541,7 +541,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -565,10 +565,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -585,7 +585,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -597,7 +597,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -619,7 +619,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -651,7 +651,7 @@ NULL NULL None 102 2 Del 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -673,10 +673,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -693,7 +693,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -705,7 +705,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -727,7 +727,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -757,7 +757,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -779,10 +779,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -799,7 +799,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -811,7 +811,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -833,7 +833,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -881,10 +881,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -901,7 +901,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -913,7 +913,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -962,7 +962,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -986,10 +986,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1004,7 +1004,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1021,7 +1021,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1040,7 +1040,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1072,7 +1072,7 @@ NULL NULL None 102 2 Del 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1094,10 +1094,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1112,7 +1112,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1129,7 +1129,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1148,7 +1148,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1175,7 +1175,7 @@ POSTHOOK: Input: default@test2 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1197,10 +1197,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1215,7 +1215,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1232,7 +1232,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1251,7 +1251,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1301,10 +1301,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1321,7 +1321,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1338,7 +1338,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1380,7 +1380,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1404,10 +1404,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1422,22 +1422,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1456,6 +1441,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1463,7 +1463,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1495,7 +1495,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1517,10 +1517,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1535,22 +1535,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1569,6 +1554,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1576,7 +1576,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1606,7 +1606,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1628,10 +1628,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1646,22 +1646,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1680,6 +1665,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1687,7 +1687,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1737,10 +1737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1757,24 +1757,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1793,6 +1776,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1827,7 +1827,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -1877,12 +1877,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 4 <- Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1899,7 +1899,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1916,7 +1916,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1925,7 +1925,26 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} + Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: test1 @@ -1942,7 +1961,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 5 + 1 Vertex 5 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1951,7 +1970,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: test2 @@ -1968,25 +1987,6 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1994,7 +1994,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1, diff --git ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out index 4539684..adcd96a 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out @@ -87,11 +87,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -111,7 +111,7 @@ STAGE PLANS: 1 _col0 (type: decimal(6,2)) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(4,2)) @@ -120,7 +120,21 @@ STAGE PLANS: value expressions: _col1 (type: decimal(4,0)) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -139,20 +153,6 @@ STAGE PLANS: Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out index 81a8546..2bc5874 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -78,7 +78,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -89,7 +89,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -134,7 +134,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -154,10 +154,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -177,7 +177,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -188,7 +188,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -210,7 +210,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) diff --git ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out index 7c504f1..b44074c 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -31,25 +31,25 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 2 => 25 + Estimated key counts: Vertex 2 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Position of Big Table: 0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 500 + Estimated key counts: Vertex 3 => 500 keys: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -276,7 +276,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src1 @@ -351,7 +351,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [src1] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -448,10 +448,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -471,7 +471,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 732 Data size: 7782 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -481,7 +481,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 805 Data size: 8560 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -492,7 +492,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src1 @@ -511,7 +511,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -552,11 +552,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -576,7 +576,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -586,7 +586,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -602,7 +602,27 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src @@ -621,7 +641,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src1 @@ -640,26 +660,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/mapreduce1.q.out ql/src/test/results/clientpositive/llap/mapreduce1.q.out index 37f92d9..9d4f7be 100644 --- ql/src/test/results/clientpositive/llap/mapreduce1.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce1.q.out @@ -33,10 +33,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -58,7 +58,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col0 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/mapreduce2.q.out ql/src/test/results/clientpositive/llap/mapreduce2.q.out index 71bbb7e..2c19819 100644 --- ql/src/test/results/clientpositive/llap/mapreduce2.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce2.q.out @@ -31,10 +31,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -55,7 +55,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col0 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/merge1.q.out ql/src/test/results/clientpositive/llap/merge1.q.out index 4bcb728..35248a4 100644 --- ql/src/test/results/clientpositive/llap/merge1.q.out +++ ql/src/test/results/clientpositive/llap/merge1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -51,7 +51,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,7 +479,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src @@ -545,7 +545,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src diff --git ql/src/test/results/clientpositive/llap/merge2.q.out ql/src/test/results/clientpositive/llap/merge2.q.out index aa0567b..f60e6f9 100644 --- ql/src/test/results/clientpositive/llap/merge2.q.out +++ ql/src/test/results/clientpositive/llap/merge2.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -51,7 +51,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,7 +479,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src @@ -545,7 +545,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src diff --git ql/src/test/results/clientpositive/llap/mergejoin.q.out ql/src/test/results/clientpositive/llap/mergejoin.q.out index 6114548..52f6940 100644 --- ql/src/test/results/clientpositive/llap/mergejoin.q.out +++ ql/src/test/results/clientpositive/llap/mergejoin.q.out @@ -13,12 +13,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -39,7 +39,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -73,25 +91,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -284,13 +284,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -323,27 +323,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -362,7 +342,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -377,7 +357,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -389,6 +369,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1401,13 +1401,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1436,26 +1436,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN DynamicValue(RS_4_a_key_min) AND DynamicValue(RS_4_a_key_max) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1474,7 +1455,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1489,7 +1470,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1501,6 +1482,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN DynamicValue(RS_4_a_key_min) AND DynamicValue(RS_4_a_key_max) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1542,13 +1542,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1567,36 +1567,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=500) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1615,7 +1586,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1630,7 +1601,36 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=500) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1683,11 +1683,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1703,23 +1703,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1738,7 +1722,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1753,6 +1737,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1790,16 +1790,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 8 (BROADCAST_EDGE) - Map 6 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 8 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 1 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (BROADCAST_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1833,60 +1833,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1902,7 +1849,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 56100 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1921,7 +1868,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1936,7 +1883,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1948,18 +1895,71 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 8 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 6 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Stage: Stage-0 Fetch Operator @@ -1997,13 +1997,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2036,27 +2036,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2075,7 +2055,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2090,7 +2070,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2102,6 +2082,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2145,15 +2145,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 3 (CONTAINS) - Map 9 <- Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Union 3 (PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS) + Vertex 9 <- Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2173,80 +2175,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: s3 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: s2 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 9 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2275,7 +2204,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2294,7 +2223,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2309,7 +2238,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2321,8 +2250,79 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Union 3 - Vertex: Union 3 + Vertex 7 + Map Operator Tree: + TableScan + alias: s3 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: s2 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2343,13 +2343,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2382,27 +2382,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2421,7 +2401,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2436,7 +2416,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2448,6 +2428,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2483,16 +2483,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 8 (BROADCAST_EDGE) - Map 6 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 8 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 1 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (BROADCAST_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2503,83 +2503,30 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Select Operator - expressions: _col0 (type: string) + expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2595,7 +2542,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 56100 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2614,7 +2561,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2629,7 +2576,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2641,7 +2588,60 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 8 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2698,15 +2698,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 3 (CONTAINS) - Map 9 <- Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Union 3 (PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS) + Vertex 9 <- Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2726,80 +2728,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: s3 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: s2 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 9 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2828,7 +2757,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2847,7 +2776,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2862,7 +2791,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2874,8 +2803,79 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Union 3 - Vertex: Union 3 + Vertex 7 + Map Operator Tree: + TableScan + alias: s3 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: s2 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2910,15 +2910,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 2 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -2937,26 +2937,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: t2 - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2981,7 +2962,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3000,7 +2981,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3015,7 +2996,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3027,7 +3008,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -3073,7 +3073,7 @@ POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### 480 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -3093,7 +3093,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -3108,7 +3108,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -3129,8 +3129,8 @@ POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -3153,8 +3153,8 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -3681,8 +3681,8 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -4203,7 +4203,7 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[30][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -4224,7 +4224,7 @@ POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[17][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[17][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join diff --git ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out index 25be543..2f9e209 100644 --- ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out +++ ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out @@ -191,10 +191,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl @@ -214,7 +214,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -251,10 +251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -274,7 +274,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -311,10 +311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl @@ -334,7 +334,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -625,10 +625,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -648,7 +648,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/metadataonly1.q.out ql/src/test/results/clientpositive/llap/metadataonly1.q.out index 27218cf..93c94ad 100644 --- ql/src/test/results/clientpositive/llap/metadataonly1.q.out +++ ql/src/test/results/clientpositive/llap/metadataonly1.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -46,7 +46,7 @@ STAGE PLANS: auto parallelism: false Execution mode: llap LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -113,10 +113,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -191,7 +191,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -332,7 +332,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -394,10 +394,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -473,7 +473,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: /test1/ds=1 [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -544,12 +544,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a2 @@ -669,7 +669,62 @@ STAGE PLANS: Truncated Path -> Alias: /test1/ds=1 [a2] /test1/ds=2 [a2] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Position of Big Table: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: test1 @@ -793,62 +848,7 @@ STAGE PLANS: Truncated Path -> Alias: /test1/ds=1 [test1] /test1/ds=2 [test1] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 5 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -883,7 +883,7 @@ STAGE PLANS: Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Stage: Stage-0 Fetch Operator @@ -946,10 +946,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1125,7 +1125,7 @@ STAGE PLANS: nullscan://null/default.test2/part_ds=1_hr=1_ [test2] nullscan://null/default.test2/part_ds=1_hr=2_ [test2] nullscan://null/default.test2/part_ds=1_hr=3_ [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1191,10 +1191,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1372,7 +1372,7 @@ STAGE PLANS: /test2/ds=1/hr=1 [test2] /test2/ds=1/hr=2 [test2] /test2/ds=1/hr=3 [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1438,10 +1438,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1564,7 +1564,7 @@ STAGE PLANS: Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] nullscan://null/default.test1/part_ds=2_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1677,10 +1677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1954,7 +1954,7 @@ STAGE PLANS: nullscan://null/default.test2/part_ds=1_hr=1_ [test2] nullscan://null/default.test2/part_ds=1_hr=2_ [test2] nullscan://null/default.test2/part_ds=1_hr=3_ [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/mrr.q.out ql/src/test/results/clientpositive/llap/mrr.q.out index 1437d5d..2118af1 100644 --- ql/src/test/results/clientpositive/llap/mrr.q.out +++ ql/src/test/results/clientpositive/llap/mrr.q.out @@ -11,11 +11,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -38,7 +38,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -52,7 +52,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -403,12 +403,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -427,27 +427,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -468,7 +448,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 609 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -486,7 +466,7 @@ STAGE PLANS: key expressions: _col1 (type: bigint), _col0 (type: string) sort order: ++ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -500,6 +480,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -837,12 +837,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -861,7 +861,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: s2 @@ -881,7 +881,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col1 (type: string), _col2 (type: string) @@ -895,7 +895,7 @@ STAGE PLANS: Statistics: Num rows: 609 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -913,7 +913,7 @@ STAGE PLANS: key expressions: _col1 (type: bigint), _col0 (type: string) sort order: ++ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1292,61 +1292,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1368,7 +1324,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1382,7 +1338,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1395,7 +1351,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1409,7 +1365,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1422,7 +1378,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1444,7 +1400,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 68 Data size: 19380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1458,7 +1414,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1475,7 +1453,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 68 Data size: 6460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1488,6 +1466,28 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 68 Data size: 6460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1686,11 +1686,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1712,27 +1712,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1749,14 +1729,14 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1770,6 +1750,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out index 73ccdf5..a040628 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out @@ -184,11 +184,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -208,7 +208,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -221,7 +221,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -234,7 +234,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -253,7 +268,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -272,21 +287,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -347,11 +347,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -371,7 +371,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -384,7 +384,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -397,7 +397,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -416,7 +431,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -435,21 +450,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -512,11 +512,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -536,7 +536,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -550,7 +550,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -566,7 +566,27 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -585,7 +605,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -605,26 +625,6 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -817,11 +817,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -841,7 +841,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -855,7 +855,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -869,7 +869,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -883,7 +883,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -900,7 +900,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -919,7 +934,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -939,7 +954,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -958,7 +973,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -977,21 +992,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1152,11 +1152,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1176,7 +1176,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -1190,7 +1190,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -1204,7 +1204,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -1218,7 +1218,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -1235,7 +1235,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -1254,7 +1269,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -1274,7 +1289,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -1293,7 +1308,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -1312,21 +1327,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1487,11 +1487,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1511,7 +1511,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -1525,7 +1525,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -1539,7 +1539,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -1553,7 +1553,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -1570,7 +1570,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -1589,7 +1604,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -1609,7 +1624,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -1628,7 +1643,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -1647,21 +1662,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1822,14 +1822,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1849,7 +1849,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: smalltbl4 @@ -1868,65 +1868,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: smalltbl3 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1947,7 +1889,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1968,7 +1910,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1989,7 +1931,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2013,7 +1955,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2028,6 +1970,64 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 7 + Map Operator Tree: + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: smalltbl3 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out index 433c701..c9dfdf9 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out @@ -21,12 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -46,7 +48,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -54,7 +56,21 @@ STAGE PLANS: Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -73,7 +89,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: x2 @@ -93,7 +109,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -101,7 +117,7 @@ STAGE PLANS: Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -120,22 +136,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -259,12 +259,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -283,7 +285,36 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y1 @@ -302,7 +333,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x2 @@ -321,7 +352,22 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y2 @@ -340,52 +386,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -509,12 +509,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Map 6 (BROADCAST_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -535,7 +537,33 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: x2 @@ -555,7 +583,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -563,7 +591,7 @@ STAGE PLANS: Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -582,34 +610,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -722,15 +722,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -750,7 +750,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -764,7 +764,64 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Vertex 7 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y2 @@ -783,7 +840,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x1 @@ -803,7 +860,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -817,7 +874,20 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y1 @@ -836,76 +906,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Reducer 7 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -990,15 +990,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1018,7 +1018,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1032,7 +1032,64 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Vertex 7 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y2 @@ -1051,7 +1108,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x1 @@ -1071,7 +1128,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1085,7 +1142,20 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y1 @@ -1104,76 +1174,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Reducer 7 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -1258,14 +1258,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1285,7 +1285,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1299,46 +1299,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1354,7 +1315,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Reducer 7 + 1 Vertex 7 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1368,7 +1329,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1381,7 +1342,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1395,7 +1356,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1492,14 +1492,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1519,7 +1519,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1533,46 +1533,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1588,7 +1549,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Reducer 7 + 1 Vertex 7 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1602,7 +1563,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1615,7 +1576,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1629,7 +1590,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1751,11 +1751,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1774,7 +1774,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 137 Data size: 2487 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1787,7 +1787,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1806,21 +1821,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1877,13 +1877,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (BROADCAST_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1899,7 +1901,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1913,7 +1915,36 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col1 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1928,7 +1959,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1943,7 +1974,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: a @@ -1959,7 +1990,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1973,37 +2004,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2019,7 +2020,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1 input vertices: - 1 Map 5 + 1 Vertex 9 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -2032,37 +2033,36 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 9 + LLAP IO: no inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: c Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out index 0cbb9de..06ade11 100644 --- ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out +++ ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out @@ -32,11 +32,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: employee @@ -57,7 +57,7 @@ STAGE PLANS: Statistics: Num rows: 36 Data size: 198 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -78,7 +78,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -173,11 +173,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: employee @@ -198,7 +198,7 @@ STAGE PLANS: Statistics: Num rows: 60 Data size: 330 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -219,7 +219,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/multi_insert.q.out ql/src/test/results/clientpositive/llap/multi_insert.q.out index 58fc759..b40c90f 100644 --- ql/src/test/results/clientpositive/llap/multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -180,7 +180,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -323,7 +323,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -466,7 +466,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -609,10 +609,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -627,7 +627,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -763,10 +763,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -781,7 +781,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -917,10 +917,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -935,7 +935,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -1071,10 +1071,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1089,7 +1089,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -1225,11 +1225,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1262,7 +1264,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1295,8 +1297,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1422,11 +1422,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1459,7 +1461,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1492,8 +1494,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1619,11 +1619,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1656,7 +1658,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1689,8 +1691,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1816,11 +1816,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1853,7 +1855,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1886,8 +1888,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -2011,7 +2011,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2114,7 +2114,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2217,7 +2217,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2320,7 +2320,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out index e8a3610..25650cd 100644 --- ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out @@ -57,7 +57,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -277,11 +277,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -374,7 +374,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -391,7 +391,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -519,11 +519,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -582,7 +582,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -599,7 +599,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Forward @@ -776,12 +776,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -901,7 +901,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -918,7 +918,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -935,7 +935,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1132,12 +1132,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -1234,7 +1234,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1251,7 +1251,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1268,7 +1268,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Forward diff --git ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out index 1441ed0..049cc7c 100644 --- ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out +++ ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out @@ -118,12 +118,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=500 width=184) - Output:["_col0"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select key, value from src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/offset_limit.q.out ql/src/test/results/clientpositive/llap/offset_limit.q.out index adfeb05..b6b29be 100644 --- ql/src/test/results/clientpositive/llap/offset_limit.q.out +++ ql/src/test/results/clientpositive/llap/offset_limit.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -56,7 +56,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index fb8e3ad..364e4e2 100644 --- ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -101,10 +101,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -121,7 +121,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -189,10 +189,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -216,7 +216,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -286,10 +286,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -313,7 +313,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -383,10 +383,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -408,7 +408,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -501,7 +501,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -576,10 +576,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -600,7 +600,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -675,10 +675,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -701,7 +701,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -795,11 +795,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -822,7 +822,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -836,7 +836,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -904,10 +904,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -925,7 +925,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -995,10 +995,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1015,7 +1015,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1133,11 +1133,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1154,7 +1154,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1172,7 +1172,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1298,13 +1298,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1321,24 +1321,7 @@ STAGE PLANS: TopN Hash Memory Usage: 2.0E-5 Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 2.0E-5 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1360,7 +1343,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1382,7 +1365,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 2.0E-5 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1400,7 +1400,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out index 4c08ee1..56bf1d4 100644 --- ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out +++ ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -76,7 +76,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -142,11 +142,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -220,7 +220,42 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -441,42 +476,7 @@ STAGE PLANS: nullscan://null/default.srcpart/part_ds=2008-04-08_hr=12_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=11_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=12_ [srcpart] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -535,11 +535,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -613,7 +615,38 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: srcpart @@ -837,7 +870,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Reducer 2 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -868,39 +901,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -928,7 +928,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 0 2000 -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain extended select * from (select key from src where false) a left outer join (select value from srcpart limit 0) b PREHOOK: type: QUERY @@ -944,11 +944,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1021,7 +1021,42 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -1242,42 +1277,7 @@ STAGE PLANS: nullscan://null/default.srcpart/part_ds=2008-04-08_hr=12_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=11_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=12_ [srcpart] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1302,7 +1302,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select key from src where false) a left outer join (select value from srcpart limit 0) b PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -1336,11 +1336,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1429,7 +1431,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1499,27 +1501,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 5 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1554,8 +1536,26 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -1588,10 +1588,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1662,7 +1662,46 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [s1] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -1733,45 +1772,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [s2] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col5, _col6 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1802,10 +1802,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1879,7 +1879,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/orc_llap.q.out ql/src/test/results/clientpositive/llap/orc_llap.q.out index 4fb3d12..246cb62 100644 --- ql/src/test/results/clientpositive/llap/orc_llap.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap.q.out @@ -81,7 +81,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: default@cross_numbers POSTHOOK: Lineage: cross_numbers.i EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -121,7 +121,7 @@ POSTHOOK: Output: default@orc_llap_small POSTHOOK: Lineage: orc_llap_small.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: orc_llap_small.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_llap_small.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select count(1) from orc_llap_small y join orc_llap_small x PREHOOK: type: QUERY @@ -137,11 +137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -155,7 +155,7 @@ STAGE PLANS: 0 1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 225 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -168,19 +168,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 15 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -195,6 +183,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 15 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -202,7 +202,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select count(1) from orc_llap_small y join orc_llap_small x PREHOOK: type: QUERY PREHOOK: Input: default@orc_llap_small @@ -245,10 +245,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -272,7 +272,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -318,10 +318,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -345,7 +345,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -391,10 +391,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -418,7 +418,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -464,11 +464,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -491,7 +491,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -513,7 +513,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -559,13 +559,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: o1 @@ -599,28 +599,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: o2 - filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: csmallint (type: smallint), cstring2 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: smallint) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -644,7 +623,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -659,7 +638,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -671,6 +650,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: o2 + filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: csmallint (type: smallint), cstring2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: smallint) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -687,7 +687,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_llap #### A masked pattern was here #### -735462183586256 -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -737,10 +737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -764,7 +764,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -810,10 +810,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -837,7 +837,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -883,10 +883,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -910,7 +910,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -956,11 +956,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -983,7 +983,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1005,7 +1005,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1051,13 +1051,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: o1 @@ -1091,28 +1091,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: o2 - filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: csmallint (type: smallint), cstring2 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: smallint) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1136,7 +1115,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1151,7 +1130,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1163,6 +1142,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: o2 + filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: csmallint (type: smallint), cstring2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: smallint) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index bc3915a..05dae47 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -259,9 +259,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -296,9 +296,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -323,9 +323,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 22 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -348,9 +348,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 16 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -373,9 +373,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -398,9 +398,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -423,9 +423,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -448,9 +448,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -473,9 +473,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1697 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -498,9 +498,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -523,9 +523,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1713 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1713 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -548,9 +548,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -573,9 +573,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 50 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 50 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -598,9 +598,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 318 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 318 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -651,9 +651,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -671,9 +671,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -705,9 +705,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -733,9 +733,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -758,9 +758,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -797,9 +797,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -822,9 +822,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -842,9 +842,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -867,9 +867,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -892,9 +892,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -917,9 +917,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 81 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 81 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -942,9 +942,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 74 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 74 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -967,9 +967,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -992,9 +992,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 13 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 13 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1017,9 +1017,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1042,9 +1042,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 7 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 7 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1067,9 +1067,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1087,9 +1087,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1107,9 +1107,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1127,9 +1127,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1152,9 +1152,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1177,9 +1177,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1202,9 +1202,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 5c54622..056c768 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -259,9 +259,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2094 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2094 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -285,9 +285,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2094 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2094 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out index 01adcd4..d5fca30 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out @@ -80,7 +80,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap_nonvector @@ -231,7 +231,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap_nonvector diff --git ql/src/test/results/clientpositive/llap/orc_merge1.q.out ql/src/test/results/clientpositive/llap/orc_merge1.q.out index ba29491..c07ec6f 100644 --- ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -145,7 +145,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -274,7 +274,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/orc_merge10.q.out ql/src/test/results/clientpositive/llap/orc_merge10.q.out index bac45ee..14727ed 100644 --- ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -145,7 +145,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -274,7 +274,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/orc_merge2.q.out ql/src/test/results/clientpositive/llap/orc_merge2.q.out index c38852a..45467e8 100644 --- ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/orc_merge3.q.out ql/src/test/results/clientpositive/llap/orc_merge3.q.out index fd79d9d..d1c351a 100644 --- ql/src/test/results/clientpositive/llap/orc_merge3.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge3.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/orc_merge4.q.out ql/src/test/results/clientpositive/llap/orc_merge4.q.out index 6e2617b..2935b62 100644 --- ql/src/test/results/clientpositive/llap/orc_merge4.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge4.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/orc_merge5.q.out ql/src/test/results/clientpositive/llap/orc_merge5.q.out index 47096bb..6a49d3e 100644 --- ql/src/test/results/clientpositive/llap/orc_merge5.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge5.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -131,7 +131,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge6.q.out ql/src/test/results/clientpositive/llap/orc_merge6.q.out index 3969a96..b19ceb6 100644 --- ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -176,7 +176,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge7.q.out ql/src/test/results/clientpositive/llap/orc_merge7.q.out index d86d2dc..6545871 100644 --- ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -210,7 +210,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out index ba29491..c07ec6f 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -145,7 +145,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -274,7 +274,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out index e5101f4..089db10 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index 6fcb45a..38877b1 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index e28ed5d..fa0e882 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -211,9 +211,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -248,9 +248,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -275,9 +275,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 22 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -300,9 +300,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 16 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -325,9 +325,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -350,9 +350,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -375,9 +375,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -400,9 +400,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -425,9 +425,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1697 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -450,9 +450,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -475,9 +475,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1713 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1713 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -500,9 +500,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -525,9 +525,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 50 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 50 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -550,9 +550,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 318 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 318 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -603,9 +603,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -623,9 +623,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -657,9 +657,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -685,9 +685,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -710,9 +710,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -749,9 +749,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -774,9 +774,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -794,9 +794,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -819,9 +819,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -844,9 +844,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -869,9 +869,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 81 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 81 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -894,9 +894,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 74 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 74 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -919,9 +919,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -944,9 +944,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 13 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 13 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -969,9 +969,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -994,9 +994,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 7 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 7 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1019,9 +1019,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1039,9 +1039,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1059,9 +1059,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1079,9 +1079,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1104,9 +1104,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1129,9 +1129,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1154,9 +1154,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1179,9 +1179,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 8400 @@ -1207,9 +1207,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -1232,9 +1232,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -1257,9 +1257,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index 4cc8984..b91561a 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -211,9 +211,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -251,9 +251,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -276,9 +276,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -301,9 +301,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -326,9 +326,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -355,9 +355,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -388,9 +388,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -407,9 +407,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -426,9 +426,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -445,9 +445,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -468,9 +468,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -501,9 +501,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -520,9 +520,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -539,9 +539,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -558,9 +558,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -581,9 +581,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -614,9 +614,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -633,9 +633,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -652,9 +652,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -671,9 +671,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -694,9 +694,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1566 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1566 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 1566 @@ -713,9 +713,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1566 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1566 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 1566 @@ -732,9 +732,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -751,9 +751,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -770,9 +770,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -789,9 +789,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -808,9 +808,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 8400 @@ -836,9 +836,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -865,9 +865,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -884,9 +884,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -907,9 +907,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -926,9 +926,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -945,9 +945,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -973,9 +973,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1002,9 +1002,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1021,9 +1021,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1044,9 +1044,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1063,9 +1063,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1086,9 +1086,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1105,9 +1105,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1128,9 +1128,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1153,9 +1153,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1182,9 +1182,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1201,9 +1201,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1220,9 +1220,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1239,9 +1239,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 diff --git ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out index 4bba265..6a70e52 100644 --- ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out @@ -126,10 +126,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -149,7 +149,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -184,10 +184,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -207,7 +207,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -314,10 +314,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -340,7 +340,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -381,10 +381,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -408,7 +408,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -582,10 +582,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -603,7 +603,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 8602 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -647,10 +647,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -669,7 +669,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 8602 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -779,10 +779,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -802,7 +802,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -857,10 +857,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -881,7 +881,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1002,11 +1002,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -1026,7 +1026,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1042,7 +1042,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 296 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1101,11 +1101,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -1126,7 +1126,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1142,7 +1142,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 296 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/parallel.q.out ql/src/test/results/clientpositive/llap/parallel.q.out index 7dba122..6212a95 100644 --- ql/src/test/results/clientpositive/llap/parallel.q.out +++ ql/src/test/results/clientpositive/llap/parallel.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -62,7 +62,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -75,7 +75,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Forward diff --git ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out index 01348a6..f985d57 100644 --- ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out @@ -118,10 +118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -176,10 +176,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -199,7 +199,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -520,10 +520,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -541,7 +541,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -585,10 +585,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -607,7 +607,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -717,10 +717,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -740,7 +740,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -795,10 +795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -819,7 +819,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -985,11 +985,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1009,7 +1009,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1025,7 +1025,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1084,11 +1084,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1109,7 +1109,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1125,7 +1125,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1196,11 +1196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1221,7 +1221,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1237,7 +1237,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: int), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/partition_multilevels.q.out ql/src/test/results/clientpositive/llap/partition_multilevels.q.out index 5082734..23fe671 100644 --- ql/src/test/results/clientpositive/llap/partition_multilevels.q.out +++ ql/src/test/results/clientpositive/llap/partition_multilevels.q.out @@ -975,10 +975,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: partition_test_multilevel @@ -1001,7 +1001,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1565,10 +1565,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: partition_test_multilevel @@ -1591,7 +1591,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/ppd_union_view.q.out ql/src/test/results/clientpositive/llap/ppd_union_view.q.out index 543d176..caaf251 100644 --- ql/src/test/results/clientpositive/llap/ppd_union_view.q.out +++ ql/src/test/results/clientpositive/llap/ppd_union_view.q.out @@ -145,11 +145,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1_new @@ -191,7 +193,7 @@ STAGE PLANS: MultiFileSpray: false Execution mode: llap LLAP IO: unknown - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1_old @@ -270,7 +272,50 @@ STAGE PLANS: name: default.t1_old Truncated Path -> Alias: /t1_old/ds=2011-10-13 [t1_old] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col1, _col3 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 5 Map Operator Tree: TableScan alias: t1_mapping @@ -349,51 +394,6 @@ STAGE PLANS: name: default.t1_mapping Truncated Path -> Alias: /t1_mapping/ds=2011-10-13 [t1_mapping] - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col3 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2 - columns.types string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -454,11 +454,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1_new @@ -548,7 +550,7 @@ STAGE PLANS: name: default.t1_new Truncated Path -> Alias: /t1_new/ds=2011-10-15 [t1_new] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1_old @@ -575,34 +577,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 5 - Map Operator Tree: - TableScan - alias: t1_mapping - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key (type: string), keymap (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: _col0 (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -645,8 +620,33 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: t1_mapping + properties: + insideView TRUE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), keymap (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: _col0 (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/ptf.q.out ql/src/test/results/clientpositive/llap/ptf.q.out index df55238..009ab91 100644 --- ql/src/test/results/clientpositive/llap/ptf.q.out +++ ql/src/test/results/clientpositive/llap/ptf.q.out @@ -27,11 +27,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -71,7 +71,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -201,12 +201,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -222,22 +222,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -254,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -281,7 +266,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -320,6 +305,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -392,10 +392,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -408,7 +408,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -516,11 +516,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -533,7 +533,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -560,7 +560,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -696,11 +696,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -713,7 +713,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -740,7 +740,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -879,11 +879,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -896,7 +896,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -931,7 +931,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1064,11 +1064,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1081,22 +1081,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1126,7 +1111,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 23062 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1144,6 +1129,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1218,11 +1218,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1237,20 +1237,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_name (type: string) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1272,7 +1259,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_name (type: string) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1376,11 +1376,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1408,7 +1408,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1435,7 +1435,7 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1550,11 +1550,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1583,7 +1583,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1611,7 +1611,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1743,11 +1743,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1760,7 +1760,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1787,7 +1787,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1921,12 +1921,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1939,7 +1939,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1989,7 +1989,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2024,7 +2024,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2166,11 +2166,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2183,7 +2183,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2210,7 +2210,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2348,12 +2348,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2366,22 +2366,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2411,7 +2396,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 13078 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2428,7 +2413,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2493,6 +2478,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2575,11 +2575,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2592,7 +2592,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2627,7 +2627,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2734,11 +2734,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2761,7 +2761,7 @@ STAGE PLANS: value expressions: _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2794,7 +2794,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2964,13 +2964,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2983,7 +2983,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3016,7 +3016,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3069,7 +3069,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.part_4 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3106,7 +3106,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: sum_window_0 (type: bigint), _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3357,12 +3357,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3375,7 +3375,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3432,7 +3432,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3467,7 +3467,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3631,13 +3631,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3650,7 +3650,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3684,7 +3684,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3711,7 +3711,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3738,7 +3738,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3898,12 +3898,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3916,7 +3916,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3950,7 +3950,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3984,7 +3984,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4144,13 +4144,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4163,7 +4163,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4197,7 +4197,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4240,7 +4240,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4268,7 +4268,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4430,12 +4430,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4448,7 +4448,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4498,7 +4498,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4533,7 +4533,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4689,12 +4689,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4707,7 +4707,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4757,7 +4757,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4785,7 +4785,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out index 6c60f98..89d7501 100644 --- ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out +++ ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out @@ -65,10 +65,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -81,7 +81,7 @@ STAGE PLANS: value expressions: origin_city_name (type: string), arr_delay (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -193,10 +193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -209,7 +209,7 @@ STAGE PLANS: value expressions: origin_city_name (type: string), arr_delay (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -311,10 +311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -334,7 +334,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col5 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/ptf_streaming.q.out ql/src/test/results/clientpositive/llap/ptf_streaming.q.out index 6013c11..ca18afe 100644 --- ql/src/test/results/clientpositive/llap/ptf_streaming.q.out +++ ql/src/test/results/clientpositive/llap/ptf_streaming.q.out @@ -27,11 +27,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -71,7 +71,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -201,12 +201,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -222,22 +222,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -254,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -281,7 +266,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -320,6 +305,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -394,11 +394,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -411,22 +411,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -456,7 +441,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 23062 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -474,6 +459,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -548,11 +548,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -580,7 +580,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -607,7 +607,7 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -722,11 +722,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -755,7 +755,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -783,7 +783,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -917,12 +917,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -935,7 +935,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -985,7 +985,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1020,7 +1020,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1156,12 +1156,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1174,7 +1174,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1224,7 +1224,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1259,7 +1259,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1395,12 +1395,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1413,7 +1413,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1463,7 +1463,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1498,7 +1498,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1638,12 +1638,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1656,22 +1656,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1701,7 +1686,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 13078 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1718,7 +1703,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1783,6 +1768,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1887,12 +1887,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1905,7 +1905,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1962,7 +1962,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1997,7 +1997,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2161,13 +2161,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2180,7 +2180,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2214,7 +2214,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2241,7 +2241,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2268,7 +2268,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2426,12 +2426,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2444,7 +2444,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2494,7 +2494,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2522,7 +2522,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out index 7d1f526..731a7d3 100644 --- ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out @@ -64,7 +64,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: rcfile_createas1a diff --git ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out index 2bb8e8a..6955d6c 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out @@ -35,7 +35,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out index 11b0b48..9ed0ee5 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: rcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out index 443014b..34a985c 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: rcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 2cc6f5d..9e4a252 100644 --- ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -100,7 +100,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -267,10 +267,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: complex_tbl_2 @@ -306,7 +306,7 @@ STAGE PLANS: tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) auto parallelism: true - Reducer 2 + Vertex 2 Needs Tagging: false Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out index 08a9bd1..53b1f88 100644 --- ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out +++ ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -30,7 +30,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -70,10 +70,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -90,7 +90,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -134,10 +134,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -154,7 +154,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -198,10 +198,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -224,7 +224,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,11 +260,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -284,26 +284,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -326,7 +307,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5580 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -346,6 +327,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -366,11 +366,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -390,26 +390,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -424,7 +405,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -438,6 +419,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -458,10 +458,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -482,7 +482,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -526,10 +526,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -550,7 +550,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2570,10 +2570,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2589,7 +2589,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2629,10 +2629,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2649,7 +2649,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2693,10 +2693,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2713,7 +2713,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2757,10 +2757,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2776,7 +2776,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2812,11 +2812,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2836,26 +2836,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2871,7 +2852,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2891,6 +2872,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2911,11 +2911,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2935,26 +2935,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2969,7 +2950,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2983,6 +2964,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3003,10 +3003,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3022,7 +3022,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3066,10 +3066,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3085,7 +3085,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/sample1.q.out ql/src/test/results/clientpositive/llap/sample1.q.out index 61a3a0f..630690e 100644 --- ql/src/test/results/clientpositive/llap/sample1.q.out +++ ql/src/test/results/clientpositive/llap/sample1.q.out @@ -27,7 +27,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s diff --git ql/src/test/results/clientpositive/llap/sample10.q.out ql/src/test/results/clientpositive/llap/sample10.q.out index a3da787..36eacaf 100644 --- ql/src/test/results/clientpositive/llap/sample10.q.out +++ ql/src/test/results/clientpositive/llap/sample10.q.out @@ -48,11 +48,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpartbucket @@ -292,7 +292,7 @@ STAGE PLANS: /srcpartbucket/ds=2008-04-08/hr=12/000000_0 [srcpartbucket] /srcpartbucket/ds=2008-04-09/hr=11/000000_0 [srcpartbucket] /srcpartbucket/ds=2008-04-09/hr=12/000000_0 [srcpartbucket] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -310,7 +310,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out index a3a54d1..656ae4b 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out @@ -526,7 +526,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_multi_ints_string_group @@ -705,7 +705,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_floating_string_group diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out index 81d7a24..7a43540 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out @@ -526,7 +526,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_multi_ints_string_group @@ -705,7 +705,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_floating_string_group diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out index cc98981..1242b44 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out @@ -66,7 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -171,7 +171,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -334,7 +334,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=5 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -434,7 +434,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -603,7 +603,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] @@ -762,7 +762,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] @@ -907,7 +907,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -1080,7 +1080,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] @@ -1195,7 +1195,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out index 3da9284..c47b927 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out @@ -144,7 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -396,7 +396,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=8 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -576,7 +576,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=4 width=4) Output:["_col0","_col1","_col2","_col3"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out index 526cd21..e87484f 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out @@ -255,7 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=10 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] @@ -456,7 +456,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] @@ -579,7 +579,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] @@ -686,7 +686,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] @@ -874,7 +874,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=13 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out index 7b420e2..b312532 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out @@ -68,7 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=99) Output:["_col0","_col1","_col2"] @@ -181,7 +181,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=114) Output:["_col0","_col1","_col2"] @@ -356,7 +356,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=370) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -600,7 +600,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=479) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] @@ -755,7 +755,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=752) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out index 86c94f7..5a52577 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out @@ -66,7 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -171,7 +171,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -334,7 +334,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=5 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -434,7 +434,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -603,7 +603,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] @@ -762,7 +762,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] @@ -907,7 +907,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -1080,7 +1080,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] @@ -1195,7 +1195,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out index 3da9284..c47b927 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out @@ -144,7 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -396,7 +396,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=8 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -576,7 +576,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=4 width=4) Output:["_col0","_col1","_col2","_col3"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out index f155b00..56b2a04 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out @@ -255,7 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=10 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] @@ -456,7 +456,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] @@ -579,7 +579,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] @@ -686,7 +686,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] @@ -874,7 +874,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=13 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out index ecc4ee6..2605475 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out @@ -68,7 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=99) Output:["_col0","_col1","_col2"] @@ -181,7 +181,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=114) Output:["_col0","_col1","_col2"] @@ -356,7 +356,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=370) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -600,7 +600,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=479) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] @@ -755,7 +755,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=752) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out index 89a66ba..390a93e 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out @@ -66,7 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -171,7 +171,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -334,7 +334,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=5 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -434,7 +434,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -603,7 +603,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] @@ -762,7 +762,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] @@ -907,7 +907,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -1080,7 +1080,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] @@ -1195,7 +1195,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out index be42c05..9370a32 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out @@ -144,7 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -396,7 +396,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=8 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -576,7 +576,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=4 width=4) Output:["_col0","_col1","_col2","_col3"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out index 03c6936..3bcb6a4 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out @@ -255,7 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=10 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] @@ -456,7 +456,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] @@ -579,7 +579,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] @@ -686,7 +686,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] @@ -874,7 +874,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=13 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out index da99110..e2cb434 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out @@ -68,7 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=20) Output:["_col0","_col1","_col2"] @@ -181,7 +181,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=21) Output:["_col0","_col1","_col2"] @@ -356,7 +356,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=80) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -600,7 +600,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=178) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] @@ -755,7 +755,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=249) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out index 1fe9a13..8cb7337 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out @@ -66,7 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -171,7 +171,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=2 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -334,7 +334,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=5 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -434,7 +434,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -603,7 +603,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] @@ -762,7 +762,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] @@ -907,7 +907,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] @@ -1080,7 +1080,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] @@ -1195,7 +1195,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out index be42c05..9370a32 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out @@ -144,7 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -396,7 +396,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=8 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -576,7 +576,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=4 width=4) Output:["_col0","_col1","_col2","_col3"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out index b8601df..e95f743 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out @@ -255,12 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=507) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -456,12 +451,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=453) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -579,12 +569,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=150) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -686,12 +671,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -874,12 +854,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out index 06d2372..980f4c7 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out @@ -68,7 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=20) Output:["_col0","_col1","_col2"] @@ -181,7 +181,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=21) Output:["_col0","_col1","_col2"] @@ -356,7 +356,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=80) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -600,7 +600,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=178) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] @@ -755,7 +755,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=249) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out index 1fe9a13..40219f5 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out @@ -66,12 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY @@ -171,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY @@ -334,12 +324,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=94) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -434,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=172) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -603,12 +583,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=182) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -762,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -907,12 +877,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=200) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1080,12 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1195,12 +1155,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out index be42c05..9370a32 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out @@ -144,7 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -396,7 +396,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=8 width=4) Output:["_col0","_col1","_col2","_col3"] @@ -576,7 +576,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap + Vertex 1 llap File Output Operator [FS_2] Select Operator [SEL_1] (rows=4 width=4) Output:["_col0","_col1","_col2","_col3"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out index bb3dd31..f4d12ed 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out @@ -255,7 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=10 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] @@ -456,7 +456,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] @@ -579,7 +579,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] @@ -686,7 +686,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] @@ -874,7 +874,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap + Vertex 1 vectorized, llap File Output Operator [FS_4] Select Operator [SEL_3] (rows=13 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out index 06d2372..86a0a03 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out @@ -68,12 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_permute_select PREHOOK: type: QUERY @@ -181,12 +176,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=21) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=21) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_string_permute_select PREHOOK: type: QUERY @@ -356,12 +346,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=80) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=6 width=80) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -600,12 +585,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=178) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=6 width=178) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -755,12 +735,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=249) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=6 width=249) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out index afb9c4f..4a2c680 100644 --- ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out +++ ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -35,7 +35,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -704,10 +704,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -727,7 +727,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -928,12 +928,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -954,7 +956,22 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -975,23 +992,6 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -2471,10 +2471,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2495,7 +2495,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3164,10 +3164,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3187,7 +3187,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3388,12 +3388,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3414,7 +3416,22 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -3435,23 +3452,6 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/semijoin.q.out ql/src/test/results/clientpositive/llap/semijoin.q.out index 28fc1d5..f8996ef 100644 --- ql/src/test/results/clientpositive/llap/semijoin.q.out +++ ql/src/test/results/clientpositive/llap/semijoin.q.out @@ -133,11 +133,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -153,31 +153,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -192,7 +168,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -206,6 +182,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -242,11 +242,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -262,31 +262,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -301,7 +277,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -315,6 +291,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -353,11 +353,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -373,31 +373,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -412,7 +388,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -426,6 +402,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -456,11 +456,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -476,31 +476,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 15) (type: boolean) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -519,7 +495,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -533,6 +509,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 15) (type: boolean) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -574,11 +574,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -594,31 +594,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((value < 'val_10') and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -633,7 +609,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -647,6 +623,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value < 'val_10') and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -680,11 +680,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -708,23 +708,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -743,7 +727,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -757,17 +741,33 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t3 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t3 #### A masked pattern was here #### POSTHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value POSTHOOK: type: QUERY @@ -790,11 +790,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -818,23 +818,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -853,7 +837,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -867,6 +851,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -897,11 +897,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -925,23 +925,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -956,7 +940,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -970,6 +954,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1005,11 +1005,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1024,31 +1024,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1063,7 +1039,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1077,6 +1053,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1126,11 +1126,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1146,31 +1146,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (2 * key) is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (2 * _col0) (type: int) - sort order: + - Map-reduce partition columns: (2 * _col0) (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1185,7 +1161,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1199,6 +1175,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (2 * key) is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (2 * _col0) (type: int) + sort order: + + Map-reduce partition columns: (2 * _col0) (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1233,11 +1233,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1253,7 +1253,43 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: int), _col3 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1269,7 +1305,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1293,42 +1329,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: int), _col3 (type: string) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1375,11 +1375,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1394,31 +1394,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1433,7 +1409,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1447,6 +1423,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1491,11 +1491,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1510,7 +1510,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Semi Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1534,7 +1565,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1558,37 +1589,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Semi Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1635,11 +1635,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1651,7 +1651,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1663,7 +1694,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1684,37 +1715,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1773,11 +1773,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1789,7 +1789,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1801,7 +1832,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1822,37 +1853,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1914,11 +1914,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1930,7 +1930,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1942,7 +1973,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1963,37 +1994,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2055,56 +2055,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: c - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2121,7 +2088,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2135,6 +2102,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2196,11 +2196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2212,7 +2212,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2233,7 +2264,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2245,37 +2276,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2339,11 +2339,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2355,7 +2355,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2376,7 +2407,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2388,37 +2419,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2493,12 +2493,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2514,43 +2514,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2567,7 +2531,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2582,7 +2546,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 196 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2596,6 +2560,42 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2663,10 +2663,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2686,7 +2686,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2710,24 +2728,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/skewjoin.q.out ql/src/test/results/clientpositive/llap/skewjoin.q.out index 1251b67..5a41ea1 100644 --- ql/src/test/results/clientpositive/llap/skewjoin.q.out +++ ql/src/test/results/clientpositive/llap/skewjoin.q.out @@ -89,10 +89,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -111,27 +111,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -154,6 +134,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -213,10 +213,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -232,7 +232,33 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + 3 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -248,7 +274,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -264,7 +290,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: d @@ -280,32 +306,6 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - Inner Join 2 to 3 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - 3 key (type: string) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -355,10 +355,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -374,7 +374,33 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + 3 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -390,7 +416,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -406,7 +432,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: d @@ -422,32 +448,6 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - Inner Join 2 to 3 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - 3 key (type: string) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -489,11 +489,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -509,23 +509,7 @@ STAGE PLANS: value expressions: key (type: string), val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key + 1) is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: key (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -545,7 +529,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -560,6 +544,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: key (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -601,11 +601,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -624,27 +624,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -668,7 +648,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -683,6 +663,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -732,11 +732,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -755,27 +755,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -799,7 +779,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -814,6 +794,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -871,11 +871,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -894,46 +894,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -959,7 +920,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -974,6 +935,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1021,11 +1021,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: k @@ -1038,20 +1038,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: v - Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE - value expressions: val (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1071,7 +1058,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1086,6 +1073,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: v + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: val (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out index 5d7f371..6fd3de7 100644 --- ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out +++ ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -104,7 +104,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -124,24 +142,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -182,10 +182,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -202,24 +202,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -237,6 +220,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -279,11 +279,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -302,26 +302,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -340,7 +321,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -355,6 +336,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -388,11 +388,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,23 +408,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -443,7 +427,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -458,6 +442,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_cache.q.out ql/src/test/results/clientpositive/llap/smb_cache.q.out index 932b211..85b17f0 100644 --- ql/src/test/results/clientpositive/llap/smb_cache.q.out +++ ql/src/test/results/clientpositive/llap/smb_cache.q.out @@ -203,34 +203,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=579 width=8) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=130 width=8) - Output:["_col0"] - Filter Operator [FIL_13] (rows=130 width=8) - predicate:userid is not null - TableScan [TS_0] (rows=130 width=8) - default@bug_201_input_b,t1,Tbl:COMPLETE,Col:NONE,Output:["userid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=527 width=8) - Output:["_col0"] - Filter Operator [FIL_14] (rows=527 width=8) - predicate:userid is not null - TableScan [TS_3] (rows=527 width=8) - default@bug_201_input_a,fa,Tbl:COMPLETE,Col:NONE,Output:["userid"] + Vertex 2 PREHOOK: query: select t1.userid, @@ -283,34 +262,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=579 width=8) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=130 width=8) - Output:["_col0"] - Filter Operator [FIL_13] (rows=130 width=8) - predicate:userid is not null - TableScan [TS_0] (rows=130 width=8) - default@bug_201_input_b,t1,Tbl:COMPLETE,Col:NONE,Output:["userid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=527 width=8) - Output:["_col0"] - Filter Operator [FIL_14] (rows=527 width=8) - predicate:userid is not null - TableScan [TS_3] (rows=527 width=8) - default@bug_201_input_a,fa,Tbl:COMPLETE,Col:NONE,Output:["userid"] + Vertex 2 PREHOOK: query: select t1.userid, diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out index 8dee9ff..72d9f8b 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out @@ -57,11 +57,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -76,22 +76,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -110,7 +95,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -125,6 +110,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -172,12 +172,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -192,22 +192,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -230,7 +215,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -244,7 +229,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -258,6 +243,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -320,12 +320,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -340,22 +340,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -378,7 +363,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -398,7 +383,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -413,6 +398,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -468,30 +468,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -510,7 +491,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -529,7 +510,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -544,6 +525,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -603,11 +603,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -626,22 +626,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -660,7 +645,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -675,6 +660,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -758,30 +758,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -800,7 +781,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -819,7 +800,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -834,6 +815,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 8) and (key < 6)) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -905,11 +905,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -928,26 +928,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -966,7 +947,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -981,6 +962,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1030,11 +1030,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1053,26 +1053,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (key + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1091,7 +1072,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1106,6 +1087,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (key + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1151,11 +1151,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1174,22 +1174,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1208,7 +1193,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1223,6 +1208,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1264,11 +1264,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1287,22 +1287,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1321,7 +1306,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1336,6 +1321,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1387,11 +1387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1410,7 +1410,43 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -1429,7 +1465,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -1448,42 +1484,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1551,11 +1551,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1574,22 +1574,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1608,7 +1593,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1623,6 +1608,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out index 20766c2..dcc7b06 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out @@ -47,11 +47,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -127,7 +127,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types int:string:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -203,66 +263,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types int:string:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -357,11 +357,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -437,7 +437,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -513,66 +573,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -615,11 +615,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -695,7 +695,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key2 (type: int), key (type: int) + 1 key2 (type: int), key (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -771,66 +831,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key2 (type: int), key (type: int) - 1 key2 (type: int), key (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -873,11 +873,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -953,7 +953,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int), value (type: string) + 1 key (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1029,66 +1089,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int), value (type: string) - 1 key (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out index a97a692..4b95abe 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out @@ -185,11 +185,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -204,7 +204,51 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + Inner Join 0 to 5 + Inner Join 0 to 6 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + 3 key (type: int) + 4 key (type: int) + 5 key (type: int) + 6 key (type: int) + Statistics: Num rows: 66 Data size: 462 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -219,7 +263,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -234,7 +278,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: d @@ -249,7 +293,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: e @@ -264,7 +308,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: f @@ -279,7 +323,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: g @@ -294,50 +338,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - Inner Join 0 to 3 - Inner Join 0 to 4 - Inner Join 0 to 5 - Inner Join 0 to 6 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - 3 key (type: int) - 4 key (type: int) - 5 key (type: int) - 6 key (type: int) - Statistics: Num rows: 66 Data size: 462 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -405,10 +405,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -492,7 +492,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -576,10 +576,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -673,7 +673,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,10 +787,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -962,7 +962,7 @@ STAGE PLANS: Statistics: Num rows: 165 Data size: 1155 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: llap - Map 18 + Vertex 18 Map Operator Tree: TableScan alias: q @@ -978,7 +978,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 19 + Vertex 19 Map Operator Tree: TableScan alias: r @@ -994,7 +994,31 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 20 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join0 to 2 + Left Outer Join0 to 3 + Left Outer Join0 to 4 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 20 Map Operator Tree: TableScan alias: s @@ -1010,7 +1034,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 21 + Vertex 21 Map Operator Tree: TableScan alias: t @@ -1026,30 +1050,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join0 to 2 - Left Outer Join0 to 3 - Left Outer Join0 to 4 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out index e2d5e4e..12be0b2 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -226,10 +226,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -249,7 +249,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -369,7 +369,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out index c3b29ea..519b106 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out index 7d7d96d..f999962 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -80,7 +80,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -96,7 +120,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -112,30 +136,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -170,10 +170,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -186,7 +186,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -199,7 +223,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -212,30 +236,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -270,10 +270,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -286,7 +286,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -299,7 +323,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -312,30 +336,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -391,7 +391,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -404,7 +428,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -417,30 +441,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -481,10 +481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -497,7 +497,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -510,7 +534,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -523,30 +547,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -592,10 +592,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -608,7 +608,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -621,7 +645,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -634,30 +658,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -710,7 +710,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -723,7 +747,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -736,30 +760,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -810,37 +810,11 @@ STAGE PLANS: key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -864,6 +838,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -904,10 +904,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -920,7 +920,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -933,7 +957,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -946,30 +970,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1012,10 +1012,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1028,7 +1028,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1041,7 +1065,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1054,30 +1078,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1114,10 +1114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1130,7 +1130,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1143,7 +1167,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1156,30 +1180,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1223,10 +1223,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1239,7 +1239,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1252,7 +1276,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1265,30 +1289,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1329,10 +1329,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1345,7 +1345,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1358,7 +1382,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1371,30 +1395,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out index 1abe220..cfc9ef3 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -80,7 +80,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -96,7 +120,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -112,30 +136,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -170,10 +170,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -186,7 +186,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -199,7 +223,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -212,30 +236,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -270,10 +270,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -286,7 +286,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -299,7 +323,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -312,30 +336,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -391,7 +391,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -404,7 +428,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -417,30 +441,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -481,10 +481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -497,7 +497,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -510,7 +534,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -523,30 +547,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -592,10 +592,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -608,7 +608,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -621,7 +645,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -634,30 +658,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -710,7 +710,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -723,7 +747,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -736,30 +760,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -810,37 +810,11 @@ STAGE PLANS: key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -864,6 +838,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -904,10 +904,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -920,7 +920,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -933,7 +957,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -946,30 +970,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1012,10 +1012,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1028,7 +1028,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1041,7 +1065,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1054,30 +1078,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1114,10 +1114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1130,7 +1130,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1143,7 +1167,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1156,30 +1180,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1223,10 +1223,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1239,7 +1239,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1252,7 +1276,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1265,30 +1289,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1329,10 +1329,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1345,7 +1345,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1358,7 +1382,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1371,30 +1395,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out index bbf9b46..6440616 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out @@ -73,10 +73,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -92,23 +92,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -131,6 +115,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -1251,10 +1251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1270,23 +1270,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1309,6 +1293,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2445,10 +2445,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2464,23 +2464,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2503,6 +2487,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2555,10 +2555,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2574,23 +2574,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2613,6 +2597,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2661,10 +2661,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2680,7 +2680,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2696,7 +2720,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -2712,30 +2736,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/sqlmerge.q.out ql/src/test/results/clientpositive/llap/sqlmerge.q.out index 486e812..2134a98 100644 --- ql/src/test/results/clientpositive/llap/sqlmerge.q.out +++ ql/src/test/results/clientpositive/llap/sqlmerge.q.out @@ -41,14 +41,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (SIMPLE_EDGE) - Reducer 6 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -61,20 +61,7 @@ STAGE PLANS: value expressions: ROW__ID (type: struct) Execution mode: llap LLAP IO: may be used (ACID table) - Map 7 - Map Operator Tree: - TableScan - alias: s - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: a (type: int) - sort order: + - Map-reduce partition columns: a (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: b (type: int) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -141,7 +128,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -156,7 +143,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.acidtbl - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -171,7 +158,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.acidtbl - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -195,7 +182,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_tmp_table - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -210,6 +197,19 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.acidtbl + Vertex 7 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: a (type: int) + sort order: + + Map-reduce partition columns: a (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: b (type: int) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-5 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/stats11.q.out ql/src/test/results/clientpositive/llap/stats11.q.out index 3eccf52..d799acd 100644 --- ql/src/test/results/clientpositive/llap/stats11.q.out +++ ql/src/test/results/clientpositive/llap/stats11.q.out @@ -313,10 +313,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -388,7 +388,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -462,55 +511,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -671,10 +671,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -746,7 +746,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -820,55 +869,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out index 8a85bf4..1bff6aa 100644 --- ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out +++ ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out @@ -101,7 +101,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -142,7 +142,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git ql/src/test/results/clientpositive/llap/stats_only_null.q.out ql/src/test/results/clientpositive/llap/stats_only_null.q.out index c905ceb..945755f 100644 --- ql/src/test/results/clientpositive/llap/stats_only_null.q.out +++ ql/src/test/results/clientpositive/llap/stats_only_null.q.out @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_null @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -141,10 +141,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_null_part @@ -164,7 +164,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/subquery_exists.q.out ql/src/test/results/clientpositive/llap/subquery_exists.q.out index 14b592b..07b6694 100644 --- ql/src/test/results/clientpositive/llap/subquery_exists.q.out +++ ql/src/test/results/clientpositive/llap/subquery_exists.q.out @@ -25,13 +25,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -47,7 +47,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -66,42 +84,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -122,7 +105,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -135,7 +118,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -291,13 +291,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -314,40 +314,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -365,7 +332,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -386,7 +369,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -399,7 +382,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -419,7 +419,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from src b @@ -447,11 +447,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -466,7 +466,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -488,25 +506,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -526,7 +526,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src b where exists @@ -1069,7 +1069,7 @@ POSTHOOK: query: insert into t values(0) POSTHOOK: type: QUERY POSTHOOK: Output: default@t POSTHOOK: Lineage: t.i EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from t where exists (select count(*) from src where 1=2) PREHOOK: type: QUERY POSTHOOK: query: explain select * from t where exists (select count(*) from src where 1=2) @@ -1083,11 +1083,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -1102,7 +1102,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1125,25 +1143,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1170,7 +1170,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from t where exists (select count(*) from src where 1=2) PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/llap/subquery_in.q.out ql/src/test/results/clientpositive/llap/subquery_in.q.out index 63432a0..eda1f23 100644 --- ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -17,11 +17,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: s1 @@ -61,25 +79,7 @@ STAGE PLANS: Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -149,13 +149,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -171,7 +171,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -191,42 +209,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -247,7 +230,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,7 +243,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -338,13 +338,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -364,20 +364,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -395,7 +382,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -439,7 +439,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -460,7 +460,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -527,15 +527,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -552,37 +552,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -604,7 +574,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -645,7 +628,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -672,7 +655,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -691,7 +674,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -708,7 +691,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -779,13 +779,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -801,7 +801,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -821,42 +839,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -877,7 +860,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -895,7 +878,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -987,13 +987,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -1013,51 +1013,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 9200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: l_orderkey - Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: l_orderkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1070,7 +1026,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1087,7 +1043,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1109,7 +1065,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 9200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int) + outputColumnNames: l_orderkey + Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: l_orderkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1169,7 +1169,7 @@ POSTHOOK: Input: default@lineitem #### A masked pattern was here #### 108570 8571 4297 1798 -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 11' is a cross product PREHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY POSTHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) @@ -1183,17 +1183,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE), Map 13 (CUSTOM_SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (PARTITION_EDGE), Vertex 13 (PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -1213,7 +1213,7 @@ STAGE PLANS: value expressions: _col1 (type: double), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: lineitem @@ -1225,58 +1225,7 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1297,7 +1246,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1310,7 +1259,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1327,7 +1291,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: double), _col3 (type: int) Statistics: Num rows: 26 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1347,7 +1311,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1362,7 +1326,43 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1389,7 +1389,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1408,7 +1408,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: int) Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1428,7 +1428,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 11' is a cross product PREHOOK: query: select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@lineitem @@ -1453,13 +1453,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1479,41 +1479,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1531,7 +1497,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1556,7 +1539,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1582,7 +1565,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1647,12 +1647,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1672,7 +1672,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col5 - 1) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -1695,25 +1713,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col5 - 1) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1739,7 +1739,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1790,12 +1790,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1815,7 +1815,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col0 * _col5) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -1838,25 +1856,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col0 * _col5) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1882,7 +1882,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1924,14 +1924,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -1947,41 +1947,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2000,7 +1966,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2015,7 +1981,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2040,7 +2023,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2057,7 +2040,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2099,12 +2099,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2124,7 +2124,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 floor(_col7) (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -2147,25 +2165,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 floor(_col7) (type: bigint) - 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2195,7 +2195,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2261,13 +2261,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2284,41 +2284,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2336,7 +2302,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2357,7 +2340,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2374,7 +2357,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2417,13 +2417,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2440,41 +2440,7 @@ STAGE PLANS: value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2492,7 +2458,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2517,7 +2500,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2534,7 +2517,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2601,13 +2601,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2624,41 +2624,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2676,7 +2642,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2697,7 +2680,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2710,7 +2693,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2777,13 +2777,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2800,45 +2800,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2856,7 +2818,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2877,7 +2856,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2890,7 +2869,28 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2957,15 +2957,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2985,51 +2985,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3052,7 +3008,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3070,7 +3026,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3088,7 +3044,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3101,7 +3077,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3127,7 +3127,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3199,17 +3199,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3225,7 +3225,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -3249,41 +3249,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: sc - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3309,7 +3275,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3322,7 +3288,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3345,7 +3311,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3363,7 +3329,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3381,7 +3347,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: sc + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3406,7 +3389,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3423,7 +3406,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3495,13 +3495,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3521,30 +3521,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3560,7 +3537,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3574,7 +3551,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3600,7 +3600,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3651,13 +3651,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3677,30 +3677,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3717,7 +3694,7 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3734,7 +3711,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3760,7 +3760,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3805,13 +3805,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Union 4 (CONTAINS) - Map 6 <- Union 4 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3831,7 +3833,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -3855,7 +3875,20 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 6 Map Operator Tree: TableScan alias: part @@ -3879,39 +3912,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -3944,14 +3944,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -3968,44 +3968,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_size < 10) (type: boolean) - Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_mfgr (type: string), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4025,7 +3988,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4039,7 +4002,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (p_size < 10) (type: boolean) + Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_mfgr (type: string), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4064,7 +4047,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4081,15 +4064,32 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) @@ -4132,14 +4132,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4156,7 +4156,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col2 (type: int) + 1 _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: pp @@ -4176,62 +4194,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string), _col2 (type: int) - 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4248,7 +4211,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4273,7 +4236,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4286,7 +4249,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4299,6 +4279,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4355,16 +4355,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4381,44 +4381,24 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: int) + Vertex 11 Map Operator Tree: TableScan alias: part @@ -4435,44 +4415,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4485,7 +4428,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4503,7 +4446,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4520,7 +4483,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 18 Data size: 4050 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4541,7 +4504,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4558,7 +4521,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 8 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4571,15 +4551,35 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select p_partkey from part where p_name in - (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size and p.p_size=part.p_size) + Vertex 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select p_partkey from part where p_name in + (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size and p.p_size=part.p_size) PREHOOK: type: QUERY PREHOOK: Input: default@part #### A masked pattern was here #### @@ -4629,15 +4629,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4654,7 +4654,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4675,64 +4675,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4750,7 +4693,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4768,7 +4711,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4791,7 +4754,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 3 Data size: 687 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4804,7 +4767,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 3 Data size: 687 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4821,7 +4801,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4860,14 +4860,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4884,7 +4884,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col2 (type: int) + 1 _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: pp @@ -4904,62 +4922,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string), _col2 (type: int) - 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4976,7 +4939,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5001,7 +4964,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5014,7 +4977,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5027,6 +5007,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5083,18 +5083,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5111,73 +5111,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5200,7 +5134,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5219,7 +5153,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5236,7 +5170,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5249,7 +5200,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5273,7 +5224,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), UDFToLong(_col5) (type: bigint) Statistics: Num rows: 8 Data size: 5016 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5291,7 +5242,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5312,7 +5279,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5332,7 +5299,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5345,6 +5329,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5376,14 +5376,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5400,41 +5400,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5452,7 +5418,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5479,7 +5462,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5498,7 +5481,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: double) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5515,7 +5498,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: int), _col0 (type: double) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5569,7 +5569,7 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: explain select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) @@ -5583,14 +5583,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5607,39 +5607,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5657,7 +5625,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5687,7 +5670,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5706,7 +5689,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5723,7 +5706,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: int), _col0 (type: int) Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5742,7 +5742,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5767,24 +5767,24 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 19 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5801,7 +5801,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: pp @@ -5817,107 +5817,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5940,7 +5840,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5961,7 +5861,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5974,7 +5891,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5997,7 +5930,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6016,7 +5949,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: string) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6034,7 +5967,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6051,7 +5984,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: string) Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6075,7 +6008,24 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 8 Data size: 5016 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6088,7 +6038,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6102,7 +6069,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToLong(_col0) (type: bigint) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6119,7 +6086,7 @@ STAGE PLANS: Map-reduce partition columns: _col5 (type: int), _col4 (type: string) Statistics: Num rows: 8 Data size: 5080 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6144,7 +6111,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6165,7 +6148,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6185,7 +6168,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6229,179 +6229,78 @@ POSTHOOK: Input: default@part 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful -48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i -49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick -65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr -78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull -86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully -90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -PREHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 19 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +PREHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 10 Map Operator Tree: TableScan alias: pp - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6428,7 +6327,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6449,7 +6348,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6462,7 +6378,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6489,7 +6422,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6508,7 +6441,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: int) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6526,7 +6459,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6543,7 +6476,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col3 (type: int) Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6567,7 +6500,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8 Data size: 5016 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6580,7 +6530,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6594,7 +6561,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6611,7 +6578,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col5 (type: int) Statistics: Num rows: 8 Data size: 5080 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6636,7 +6603,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6657,7 +6640,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6677,7 +6660,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6748,12 +6748,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -6773,25 +6773,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6809,7 +6791,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6830,7 +6830,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6975,14 +6975,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -6999,41 +6999,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: tt - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7051,7 +7017,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: tt + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7078,7 +7061,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7097,7 +7080,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7110,7 +7093,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/subquery_multi.q.out ql/src/test/results/clientpositive/llap/subquery_multi.q.out index bbdce1d..44c71e3 100644 --- ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -88,13 +88,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -114,47 +114,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_size is not null (type: boolean) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -171,7 +131,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -189,7 +149,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -202,7 +182,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_brand is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -257,7 +257,7 @@ POSTHOOK: Input: default@part_null 78487 NULL Manufacturer#6 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) @@ -271,15 +271,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -299,64 +299,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -374,7 +317,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -389,7 +332,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -406,7 +349,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -431,7 +374,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -444,7 +407,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -456,6 +439,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -463,7 +463,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -472,7 +472,7 @@ POSTHOOK: query: select * from part_null where p_name IN (select p_name from par POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) @@ -486,15 +486,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -514,64 +514,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_type) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -589,7 +532,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -604,7 +547,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -621,7 +564,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -646,9 +589,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reduce Operator Tree: + LLAP IO: no inputs + Vertex 6 + Execution mode: llap + Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial @@ -659,7 +622,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_type) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -671,6 +654,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -678,7 +678,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -713,7 +713,7 @@ POSTHOOK: Input: default@part_null 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) @@ -727,15 +727,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -755,67 +755,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -833,7 +773,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -848,7 +788,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -865,7 +805,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -890,7 +830,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_brand is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -903,7 +863,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -915,6 +895,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -922,7 +922,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -931,7 +931,7 @@ POSTHOOK: query: select * from part_null where p_brand IN (select p_brand from p POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) @@ -945,15 +945,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -971,64 +971,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c (type: char(2)) - outputColumnNames: c - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count(), count(c) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: c (type: char(2)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(2)) - sort order: + - Map-reduce partition columns: _col0 (type: char(2)) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1041,7 +984,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1058,7 +1001,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 5 Data size: 3341 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1082,7 +1025,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 2 Data size: 1470 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1100,7 +1043,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c (type: char(2)) + outputColumnNames: c + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count(), count(c) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1112,7 +1075,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: c (type: char(2)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(2)) + sort order: + + Map-reduce partition columns: _col0 (type: char(2)) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1130,22 +1110,42 @@ STAGE PLANS: Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: boolean) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) -PREHOOK: type: QUERY -PREHOOK: Input: default@part_null -PREHOOK: Input: default@tempty -#### A masked pattern was here #### -POSTHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@part_null + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_brand is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_null +PREHOOK: Input: default@tempty +#### A masked pattern was here #### +POSTHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_null POSTHOOK: Input: default@tempty #### A masked pattern was here #### 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr @@ -1175,7 +1175,7 @@ POSTHOOK: Input: default@tempty 78487 NULL Manufacturer#6 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) @@ -1189,13 +1189,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1215,46 +1215,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tnull - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1269,7 +1230,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1287,7 +1248,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1300,7 +1281,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tnull + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1320,7 +1320,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1357,7 +1357,7 @@ POSTHOOK: Input: default@tnull 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) @@ -1371,13 +1371,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1397,46 +1397,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_size is not null (type: boolean) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1451,7 +1412,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1469,7 +1430,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1482,7 +1463,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1502,7 +1502,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1513,7 +1513,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null POSTHOOK: Input: default@tempty #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) @@ -1527,13 +1527,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1553,46 +1553,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1607,7 +1568,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1632,7 +1593,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1645,7 +1626,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1664,7 +1664,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1701,8 +1701,8 @@ POSTHOOK: Input: default@tempty 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) @@ -1716,35 +1716,35 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE), Reducer 21 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (CUSTOM_SIMPLE_EDGE), Reducer 26 (CUSTOM_SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE) - Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 40 (SIMPLE_EDGE) - Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) - Reducer 34 <- Reducer 33 (SIMPLE_EDGE) - Reducer 36 <- Map 35 (SIMPLE_EDGE) - Reducer 38 <- Map 37 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) - Reducer 40 <- Map 39 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (PARTITION_EDGE), Vertex 21 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (PARTITION_EDGE), Vertex 26 (PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) + Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) + Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) + Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) + Vertex 40 <- Vertex 39 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1761,7 +1761,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -1778,215 +1778,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 27 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 32 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 35 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 37 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 39 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2003,7 +1795,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2030,7 +1822,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2045,7 +1837,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2070,7 +1879,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2087,7 +1896,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2108,7 +1932,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2121,7 +1945,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2138,7 +1962,28 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2151,7 +1996,22 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2172,7 +2032,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2185,7 +2045,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2198,7 +2079,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2215,7 +2113,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2236,7 +2134,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2253,7 +2151,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 6 Data size: 1633 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2271,7 +2169,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 31 + Vertex 31 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2288,7 +2186,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 33 + Vertex 32 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 33 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2313,7 +2228,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2330,7 +2245,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 36 + Vertex 35 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 36 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2343,7 +2275,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 38 + Vertex 37 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 38 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2356,7 +2305,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 39 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2381,7 +2347,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 40 + Vertex 40 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2394,7 +2360,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2419,7 +2402,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2436,7 +2419,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2456,8 +2456,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2507,17 +2507,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -2537,84 +2537,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2635,7 +2558,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 17 Data size: 3581 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2648,7 +2571,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1685 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2661,7 +2604,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2678,7 +2621,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2696,7 +2639,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2709,7 +2672,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2730,7 +2710,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 28 Data size: 5605 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2743,6 +2723,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 14 Data size: 2802 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2786,8 +2786,8 @@ POSTHOOK: Input: default@part_null 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) @@ -2795,164 +2795,58 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE), Reducer 21 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (CUSTOM_SIMPLE_EDGE), Reducer 26 (CUSTOM_SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE) - Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 40 (SIMPLE_EDGE) - Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) - Reducer 34 <- Reducer 33 (SIMPLE_EDGE) - Reducer 36 <- Map 35 (SIMPLE_EDGE) - Reducer 38 <- Map 37 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) - Reducer 40 <- Map 39 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) - sort order: +++ - Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_container (type: string) - outputColumnNames: p_type, p_container - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (PARTITION_EDGE), Vertex 21 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (PARTITION_EDGE), Vertex 26 (PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) + Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) + Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) + Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) + Vertex 40 <- Vertex 39 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + alias: part_null + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_container (type: string) - outputColumnNames: p_type, p_container - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) + sort order: +++ + Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 27 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -2969,109 +2863,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 32 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 35 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 37 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 39 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3088,7 +2880,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3115,7 +2907,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3130,7 +2922,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3155,7 +2964,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3172,7 +2981,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3193,7 +3017,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3206,7 +3030,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3223,7 +3047,28 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 5 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_container (type: string) + outputColumnNames: p_type, p_container + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3236,7 +3081,22 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3257,7 +3117,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3270,7 +3130,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_container (type: string) + outputColumnNames: p_type, p_container + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3283,7 +3164,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3300,7 +3198,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3321,7 +3219,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3338,7 +3236,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col3 (type: string) Statistics: Num rows: 5 Data size: 3939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col13 (type: bigint), _col14 (type: bigint) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3356,7 +3254,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 31 + Vertex 31 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3373,7 +3271,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 33 + Vertex 32 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 33 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3398,7 +3313,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3415,7 +3330,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 36 + Vertex 35 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 36 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3428,7 +3360,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 38 + Vertex 37 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 38 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3441,7 +3390,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 39 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3466,7 +3432,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 40 + Vertex 40 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3479,7 +3445,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3504,7 +3487,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 317 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3521,7 +3504,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 317 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3541,8 +3541,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3592,23 +3592,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 22 <- Map 21 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -3625,41 +3625,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -3676,75 +3642,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3771,7 +3669,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3786,7 +3684,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 223 Data size: 892 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3799,7 +3714,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3820,7 +3752,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3838,7 +3770,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 223 Data size: 892 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3855,7 +3787,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: int) Statistics: Num rows: 245 Data size: 981 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 19 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3872,7 +3821,7 @@ STAGE PLANS: Map-reduce partition columns: _col5 (type: int) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3885,7 +3834,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 22 + Vertex 21 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 22 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3898,7 +3864,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3915,7 +3881,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col5 (type: int) Statistics: Num rows: 245 Data size: 981 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3940,7 +3906,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3961,7 +3944,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 28 Data size: 6435 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3974,7 +3957,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4043,17 +4043,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -4070,7 +4070,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4087,7 +4087,20 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 12 Map Operator Tree: TableScan alias: part @@ -4104,54 +4117,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col2 (type: string), _col1 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4164,7 +4130,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4182,7 +4148,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col1 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4199,7 +4182,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4220,7 +4203,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4233,7 +4216,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4258,7 +4258,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4318,7 +4318,7 @@ POSTHOOK: Input: default@part_null 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 8' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4338,22 +4338,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Map 15 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 8 <- Reducer 17 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Reducer 20 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 17 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 9 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -4373,7 +4373,20 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: lineitem @@ -4393,120 +4406,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4519,7 +4419,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4540,7 +4440,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4553,7 +4453,47 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 15 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_quantity (type: double) + outputColumnNames: l_quantity + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(l_quantity) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4570,7 +4510,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_quantity (type: double) + outputColumnNames: l_quantity + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(l_quantity) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4588,7 +4548,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4601,7 +4561,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4619,7 +4579,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4636,7 +4596,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col4 (type: int) Statistics: Num rows: 5 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4658,7 +4618,47 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (l_shipmode = 'AIR') (type: boolean) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4673,7 +4673,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: double), _col4 (type: int) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4690,7 +4690,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: double) Statistics: Num rows: 14 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col4 (type: int), _col5 (type: bigint), _col6 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4725,7 +4725,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 8' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4765,21 +4765,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -4795,7 +4795,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -4815,74 +4815,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4905,7 +4838,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4931,7 +4864,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4944,7 +4877,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4965,7 +4914,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4978,7 +4927,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4991,7 +4957,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5014,7 +4980,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5032,7 +4998,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: bigint) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5050,7 +5016,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5071,7 +5054,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5084,7 +5067,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5419,7 +5419,7 @@ POSTHOOK: Input: default@src 431 val_431 3 430 val_430 3 417 val_417 3 -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') @@ -5433,13 +5433,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5456,42 +5456,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_type <> '1') (type: boolean) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5509,7 +5474,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5530,7 +5510,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5543,7 +5523,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (p_type <> '1') (type: boolean) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5562,7 +5562,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git ql/src/test/results/clientpositive/llap/subquery_notin.q.out ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 48fe336..064dae6 100644 --- ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from src @@ -24,13 +24,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,46 +45,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(key) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -101,7 +62,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -126,7 +87,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(key) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -138,7 +118,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -163,7 +163,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where src.key not in ( select key from src s1 where s1.key > '2') @@ -324,21 +324,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -355,55 +355,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: p_name (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -417,24 +369,7 @@ STAGE PLANS: value expressions: p_name (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -475,7 +410,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -496,7 +431,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -514,7 +449,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -531,7 +466,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -544,7 +496,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -557,7 +526,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -574,7 +543,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -599,7 +568,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: p_name (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -640,7 +623,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -667,7 +650,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -682,7 +665,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -740,7 +740,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select p_name, p_size from @@ -768,16 +768,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -792,33 +792,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -836,7 +810,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -853,7 +827,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col1) (type: double) Statistics: Num rows: 26 Data size: 3666 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -878,7 +852,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -922,7 +909,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -939,7 +926,20 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -983,7 +983,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1008,7 +1008,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -1080,27 +1080,27 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 22 <- Reducer 21 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Map 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1117,7 +1117,20 @@ STAGE PLANS: value expressions: _col0 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: part @@ -1130,114 +1143,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 23 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1278,7 +1184,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1305,7 +1211,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1326,7 +1232,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1339,7 +1262,20 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1380,7 +1316,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1407,7 +1343,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1431,7 +1367,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 8 Data size: 1848 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: int) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1450,7 +1386,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1468,7 +1404,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 22 + Vertex 22 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1485,7 +1421,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 24 + Vertex 23 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1498,7 +1451,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1511,7 +1481,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1528,7 +1498,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 8 Data size: 1912 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col6 (type: bigint), _col7 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1553,7 +1523,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1593,7 +1576,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 784 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1614,7 +1597,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1634,6 +1617,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1679,7 +1679,7 @@ Manufacturer#5 almond antique medium spring khaki 6 Manufacturer#5 almond azure blanched chiffon midnight 23 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select li.l_partkey, count(*) from lineitem li where li.l_linenumber = 1 and @@ -1712,7 +1712,7 @@ POSTHOOK: Input: default@lineitem 139636 1 175839 1 182052 1 -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where not src.key in ( select key from src s1 where s1.key > '2') @@ -1874,7 +1874,7 @@ POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v POSTHOOK: Lineage: T2_v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) @@ -1892,13 +1892,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1918,7 +1918,49 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4 + Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((_col1 = 0) or (_col4 is null and _col0 is not null and (_col2 >= _col1))) (type: boolean) + Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -1943,7 +1985,19 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: src @@ -1969,61 +2023,7 @@ STAGE PLANS: Statistics: Num rows: 35 Data size: 6440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((_col1 = 0) or (_col4 is null and _col0 is not null and (_col2 >= _col1))) (type: boolean) - Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2048,7 +2048,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) PREHOOK: type: QUERY @@ -2076,19 +2076,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2108,92 +2108,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2218,7 +2133,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2236,7 +2151,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2253,7 +2168,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2266,7 +2198,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2279,7 +2228,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2296,7 +2245,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2321,7 +2270,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2348,7 +2314,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2363,7 +2329,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2376,6 +2359,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2414,7 +2414,7 @@ POSTHOOK: Input: default@part 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) @@ -2428,15 +2428,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2451,53 +2451,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2514,7 +2468,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2539,29 +2493,52 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2573,7 +2550,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2596,7 +2596,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2621,7 +2621,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2646,7 +2646,7 @@ POSTHOOK: Input: default@part 48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) @@ -2660,15 +2660,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2683,53 +2683,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_partkey (type: int) - outputColumnNames: p_type, p_partkey - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_partkey (type: int) - outputColumnNames: p_type, p_partkey - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2746,7 +2700,7 @@ STAGE PLANS: Map-reduce partition columns: (_col0 * _col5) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2771,7 +2725,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_partkey (type: int) + outputColumnNames: p_type, p_partkey + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2793,7 +2770,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2805,7 +2782,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_partkey (type: int) + outputColumnNames: p_type, p_partkey + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2828,7 +2828,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2853,7 +2853,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2900,20 +2900,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -2930,58 +2930,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -2998,24 +2947,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3036,7 +2968,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3054,7 +2986,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3071,7 +3003,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3084,7 +3033,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3098,7 +3064,7 @@ STAGE PLANS: Map-reduce partition columns: (_col0 + 100) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3115,7 +3081,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 26 Data size: 3666 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3140,7 +3106,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3155,17 +3121,34 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string), _col0 (type: int) outputColumnNames: _col2, _col0 @@ -3182,7 +3165,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3197,7 +3180,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3226,7 +3226,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### 26 -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) @@ -3240,15 +3240,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3263,53 +3263,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3326,7 +3280,7 @@ STAGE PLANS: Map-reduce partition columns: floor(_col7) (type: bigint) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3351,7 +3305,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_retailprice (type: double) + outputColumnNames: p_type, p_retailprice + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_retailprice) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3373,7 +3350,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3385,7 +3362,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_retailprice (type: double) + outputColumnNames: p_type, p_retailprice + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_retailprice) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3408,7 +3408,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3433,7 +3433,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3456,19 +3456,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3485,92 +3485,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3591,7 +3506,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3609,7 +3524,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3626,7 +3541,24 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3639,7 +3571,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3652,7 +3601,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3669,7 +3618,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col5 (type: int), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3694,7 +3643,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3721,7 +3687,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3740,7 +3706,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col0 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3753,6 +3736,23 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3805,19 +3805,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3834,92 +3834,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3940,7 +3855,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3958,7 +3873,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3975,7 +3890,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col4 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3988,7 +3920,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4001,7 +3950,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4018,7 +3967,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4043,7 +3992,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4070,7 +4036,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4085,7 +4051,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4098,6 +4081,23 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4126,19 +4126,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4155,7 +4155,67 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (UDFToDouble(_col1) + 2.0) (type: double), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: double), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: boolean) + Vertex 12 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col3 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col3 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) + Vertex 13 Map Operator Tree: TableScan alias: part @@ -4172,148 +4232,37 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 14 Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 15 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: p_brand (type: string) + keys: p_type (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (UDFToDouble(_col1) + 2.0) (type: double), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: double), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: double) - 1 UDFToDouble(_col0) (type: double) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col3 (type: string), _col1 (type: string) - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) - Reducer 14 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4327,7 +4276,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4344,7 +4293,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: string) Statistics: Num rows: 26 Data size: 8658 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4369,7 +4318,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4396,7 +4362,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4411,7 +4377,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4424,6 +4407,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4452,19 +4452,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4481,100 +4481,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4595,7 +4502,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4613,7 +4520,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 9 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4630,7 +4537,28 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: int) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4643,7 +4571,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4656,7 +4601,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4673,7 +4618,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 26 Data size: 6370 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4698,7 +4643,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4725,7 +4687,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4740,7 +4702,28 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4753,6 +4736,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4794,7 +4794,7 @@ almond azure aquamarine papaya violet almond antique medium spring khaki almond aquamarine sandy cyan gainsboro almond antique olive coral navajo -Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY POSTHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) @@ -4808,17 +4808,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -4833,7 +4833,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -4857,44 +4857,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(key) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4920,7 +4883,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4933,7 +4896,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4950,7 +4913,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4980,7 +4943,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4998,7 +4961,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5016,7 +4979,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(key) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5028,7 +5011,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5053,7 +5053,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -5106,23 +5106,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 21 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -5139,82 +5139,20 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: sc - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: sc @@ -5230,38 +5168,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5286,7 +5194,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5304,7 +5212,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5321,7 +5229,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: string) Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5334,7 +5259,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5347,7 +5289,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 19 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5364,7 +5330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5390,7 +5356,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5403,7 +5369,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5433,7 +5399,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5451,7 +5417,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5469,7 +5435,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: sc + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5496,7 +5479,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 21115 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5511,6 +5494,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 21115 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5557,7 +5557,7 @@ POSTHOOK: Input: default@src 431 3 430 3 417 3 -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand @@ -5571,16 +5571,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5595,53 +5595,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5659,7 +5613,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5676,7 +5630,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5699,7 +5653,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -5713,7 +5667,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5735,7 +5712,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5747,7 +5724,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5777,7 +5777,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5802,7 +5802,7 @@ POSTHOOK: Input: default@part 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 @@ -5816,16 +5816,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5840,53 +5840,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5904,7 +5858,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5921,7 +5875,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5945,24 +5899,47 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 4 + Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 4 - Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5984,7 +5961,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5996,7 +5973,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6026,7 +6026,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6039,7 +6039,7 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY POSTHOOK: query: explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) @@ -6053,17 +6053,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 11 <- Union 9 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Union 9 (CONTAINS) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 5 (PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) + Vertex 8 <- Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 5 + Vertex: Union 5 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -6078,7 +6082,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) + Vertex 11 Map Operator Tree: TableScan alias: part @@ -6099,7 +6121,49 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col5 + Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) (type: boolean) + Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: part @@ -6119,7 +6183,19 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 7 Map Operator Tree: TableScan alias: part @@ -6139,7 +6215,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: part @@ -6160,82 +6236,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col5 - Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) (type: boolean) - Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Union 5 - Vertex: Union 5 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -6243,7 +6243,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6767,20 +6767,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -6796,93 +6796,25 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + LLAP IO: no inputs + Vertex 10 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6903,7 +6835,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6921,7 +6853,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6938,7 +6870,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6951,7 +6900,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6965,7 +6931,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble((_col0 + 100)) (type: double) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6982,7 +6948,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7007,7 +6973,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7022,7 +6988,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7049,7 +7032,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7064,7 +7047,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7124,7 +7124,7 @@ POSTHOOK: query: INSERT INTO t2 VALUES (null), (2), (100) POSTHOOK: type: QUERY POSTHOOK: Output: default@t2 POSTHOOK: Lineage: t2.c1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) PREHOOK: type: QUERY POSTHOOK: query: explain SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) @@ -7138,13 +7138,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7159,44 +7159,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: c1 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(c1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7213,7 +7176,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 381 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7238,7 +7201,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: c1 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(c1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7250,7 +7233,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7275,7 +7275,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) PREHOOK: type: QUERY PREHOOK: Input: default@t1 @@ -7299,19 +7299,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7328,92 +7328,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7434,7 +7349,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7452,7 +7367,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: char(100)), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7469,7 +7384,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: char(100)) Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7483,7 +7415,24 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: char(100)) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7496,7 +7445,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7513,7 +7462,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7538,7 +7487,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7561,7 +7527,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7576,7 +7542,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7590,6 +7573,23 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: char(100)) + Vertex 9 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -7670,19 +7670,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7699,92 +7699,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: b (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7805,7 +7720,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7823,7 +7738,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7840,7 +7755,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col3 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7853,7 +7785,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: b (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7866,7 +7815,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7883,7 +7832,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7908,7 +7857,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7935,7 +7901,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7950,7 +7916,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7963,6 +7946,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8029,133 +8029,48 @@ POSTHOOK: Output: default@fixob POSTHOOK: Lineage: fixob.i EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: fixob.j EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col2, type:string, comment:), ] PREHOOK: query: explain select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: fixob Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8176,7 +8091,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8194,7 +8109,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8211,7 +8126,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8224,7 +8156,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8237,7 +8186,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8254,7 +8203,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8279,7 +8228,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t7 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8306,7 +8272,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8321,7 +8287,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8334,6 +8317,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t7 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8397,108 +8397,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Vertex 1 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 @@ -8511,7 +8426,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8532,7 +8447,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8550,7 +8465,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8567,7 +8482,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8580,7 +8512,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8593,7 +8542,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8610,7 +8559,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8635,7 +8584,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8662,7 +8628,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8677,7 +8643,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8690,6 +8673,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8721,108 +8721,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 + Vertex 1 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 @@ -8835,7 +8750,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8856,7 +8771,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8874,7 +8789,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8891,7 +8806,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8904,7 +8836,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8917,7 +8866,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8934,7 +8883,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8959,7 +8908,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8986,7 +8952,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9001,7 +8967,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9014,6 +8997,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -9030,7 +9030,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t #### A masked pattern was here #### 7 -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select t.i from t where t.j NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY POSTHOOK: query: explain select t.i from t where t.j NOT IN (select t1.i from t t1 ) @@ -9044,13 +9044,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -9065,44 +9065,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(i) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9119,7 +9082,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 3 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9144,7 +9107,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(i) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9156,7 +9139,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9181,7 +9181,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -9192,7 +9192,7 @@ POSTHOOK: Input: default@t #### A masked pattern was here #### 1 4 -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select t.i from t where t.i NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY POSTHOOK: query: explain select t.i from t where t.i NOT IN (select t1.i from t t1 ) @@ -9206,13 +9206,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -9227,44 +9227,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(i) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9281,7 +9244,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9306,7 +9269,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(i) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9318,7 +9301,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9343,7 +9343,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select t.i from t where t.i NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -9356,8 +9356,8 @@ PREHOOK: query: drop table t1 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1 POSTHOOK: type: DROPTABLE -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 10' is a cross product PREHOOK: query: explain select * from src b where b.key not in @@ -9383,19 +9383,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Reducer 14 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -9412,94 +9412,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9527,7 +9440,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1656 Data size: 294768 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9545,7 +9458,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1656 Data size: 301392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9562,7 +9475,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 1656 Data size: 301392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9574,7 +9504,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9587,7 +9534,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9604,7 +9551,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9629,7 +9576,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9659,7 +9624,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 7704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9674,7 +9639,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 7704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9686,6 +9668,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) + Vertex 9 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -9693,8 +9693,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 10' is a cross product PREHOOK: query: select * from src b where b.key not in diff --git ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out index 7d9d77c..b895a5b 100644 --- ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out +++ ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out @@ -6,8 +6,8 @@ POSTHOOK: query: CREATE TABLE table_7 (int_col INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@table_7 -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain SELECT (t1.int_col) * (t1.int_col) AS int_col @@ -55,13 +55,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: table_7 @@ -73,50 +73,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: table_7 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count(), count(false) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: table_7 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: false (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -131,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -156,7 +113,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count(), count(false) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -175,7 +153,29 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: false (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/subquery_scalar.q.out ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index a1a74a7..29ee36b 100644 --- ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -75,8 +75,8 @@ POSTHOOK: Lineage: part_null.p_partkey EXPRESSION [(values__tmp__table__2)values POSTHOOK: Lineage: part_null.p_retailprice EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col8, type:string, comment:), ] POSTHOOK: Lineage: part_null.p_size EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col6, type:string, comment:), ] POSTHOOK: Lineage: part_null.p_type SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col5, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part where p_size > (select avg(p_size) from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size > (select avg(p_size) from part_null) @@ -90,13 +90,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -111,47 +111,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: avg(p_size) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -166,7 +126,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16328 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -191,7 +151,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -214,7 +194,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_size (type: int) + outputColumnNames: p_size + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: avg(p_size) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -233,8 +233,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part where p_size > (select avg(p_size) from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -256,8 +256,8 @@ POSTHOOK: Input: default@part_null 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part where p_size > (select * from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -268,8 +268,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@tempty #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part where p_size > (select * from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size > (select * from tempty) @@ -283,12 +283,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 6 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -303,40 +303,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c (type: char(2)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: char(2)) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -351,7 +318,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -376,7 +343,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -392,6 +377,21 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c (type: char(2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: char(2)) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -399,7 +399,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -435,43 +435,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is null (type: boolean) - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is null (type: boolean) - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -495,7 +459,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is null (type: boolean) + Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -511,6 +496,21 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is null (type: boolean) + Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -518,7 +518,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where p_name = (select p_name from part_null where p_name is null) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -529,8 +529,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part where (select i from tnull limit 1) is null PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (select i from tnull limit 1) is null @@ -544,14 +544,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -566,41 +566,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: tnull - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: tnull - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -615,7 +581,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -640,7 +606,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: tnull + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Limit @@ -655,7 +636,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -671,11 +652,30 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int) + Vertex 7 + Map Operator Tree: + TableScan + alias: tnull + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Limit @@ -692,8 +692,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part where (select i from tnull limit 1) is null PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -730,7 +730,7 @@ POSTHOOK: Input: default@tnull 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (select max(p_name) from part_null) is not null PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (select max(p_name) from part_null) is not null @@ -744,12 +744,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -764,47 +764,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(p_name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -824,7 +784,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -847,7 +827,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(p_name) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -870,7 +870,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (select max(p_name) from part_null) is not null PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -907,10 +907,10 @@ POSTHOOK: Input: default@part_null 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[57][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[57][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 4' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 5' is a cross product PREHOOK: query: explain select * from part where p_size between (select min(p_size) from part) and (select avg(p_size) from part) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size between (select min(p_size) from part) and (select avg(p_size) from part) @@ -924,17 +924,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE) - Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (PARTITION_EDGE) + Vertex 13 <- Vertex 12 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) + Vertex 4 <- Vertex 11 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 13 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -949,47 +949,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(p_size) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -1009,27 +969,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1052,7 +992,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_size (type: int) + outputColumnNames: p_size + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(p_size) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1064,7 +1024,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1079,7 +1039,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1094,7 +1054,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16198 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1109,7 +1069,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16198 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1134,7 +1094,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1157,7 +1137,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_size (type: int) + outputColumnNames: p_size + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1176,10 +1176,10 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[57][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[57][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 4' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 5' is a cross product PREHOOK: query: select * from part where p_size between (select min(p_size) from part) and (select avg(p_size) from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -1203,8 +1203,8 @@ POSTHOOK: Input: default@part 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select p_mfgr, p_name, p_size from part where part.p_size > (select first_value(p_size) over(partition by p_mfgr order by p_size) as fv from part order by fv limit 1) @@ -1222,16 +1222,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1246,31 +1246,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1284,7 +1260,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1299,7 +1275,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1324,7 +1300,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1359,7 +1347,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1376,7 +1364,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1392,7 +1380,19 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1435,8 +1435,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select p_mfgr, p_name, p_size from part where part.p_size > (select first_value(p_size) over(partition by p_mfgr order by p_size) as fv from part order by fv limit 1) @@ -1474,8 +1474,8 @@ Manufacturer#5 almond antique medium spring khaki 6 Manufacturer#5 almond antique sky peru orange 2 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part where (p_partkey*p_size) <> (select min(p_partkey) from part) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_partkey*p_size) <> (select min(p_partkey) from part) @@ -1489,13 +1489,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1510,47 +1510,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1565,7 +1525,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1590,7 +1550,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1613,7 +1593,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1632,8 +1632,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part where (p_partkey*p_size) <> (select min(p_partkey) from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -1681,14 +1681,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -1705,41 +1705,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1764,7 +1730,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1779,7 +1745,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1806,7 +1789,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1825,7 +1808,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: boolean) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1867,13 +1867,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1890,41 +1890,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1949,7 +1915,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1976,7 +1959,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 34 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1995,7 +1978,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: double), _col1 (type: boolean) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2026,8 +2026,8 @@ POSTHOOK: Input: default@part POSTHOOK: Input: default@part_null #### A masked pattern was here #### 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 4' is a cross product PREHOOK: query: explain select * from part where p_size BETWEEN (select min(p_size) from part_null where part_null.p_type = part.p_type) AND (select max(p_size) from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size BETWEEN (select min(p_size) from part_null where part_null.p_type = part.p_type) AND (select max(p_size) from part_null) @@ -2041,17 +2041,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE) - Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (PARTITION_EDGE) + Vertex 13 <- Vertex 12 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 11 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 13 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2068,7 +2068,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part_null @@ -2088,61 +2088,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(p_size) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2165,8 +2111,28 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 13 - Execution mode: llap + Vertex 12 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_size (type: int) + outputColumnNames: p_size + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(p_size) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 13 + Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0) @@ -2177,7 +2143,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2192,7 +2158,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 28 Data size: 17703 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col10 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2207,7 +2173,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 28 Data size: 17955 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col10 (type: boolean) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2232,7 +2198,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2259,7 +2242,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 34 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2278,7 +2261,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: boolean) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2298,8 +2298,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part where p_size BETWEEN (select min(p_size) from part_null where part_null.p_type = part.p_type) AND (select max(p_size) from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2336,7 +2336,7 @@ POSTHOOK: Input: default@part_null 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part where p_size >= (select min(p_size) from part_null where part_null.p_type = part.p_type) AND p_retailprice <= (select max(p_retailprice) from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size >= (select min(p_size) from part_null where part_null.p_type = part.p_type) AND p_retailprice <= (select max(p_retailprice) from part_null) @@ -2350,16 +2350,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (PARTITION_EDGE) + Vertex 12 <- Vertex 11 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 10 (PARTITION_EDGE), Vertex 12 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2376,81 +2376,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 407 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_retailprice (type: double) - outputColumnNames: p_retailprice - Statistics: Num rows: 407 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(p_retailprice) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2473,7 +2399,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 407 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_retailprice (type: double) + outputColumnNames: p_retailprice + Statistics: Num rows: 407 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(p_retailprice) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2485,7 +2431,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: double) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2507,7 +2453,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 9 Data size: 5690 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2534,7 +2480,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 31 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2561,7 +2524,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 34 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2580,7 +2543,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: boolean) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2593,6 +2573,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2600,7 +2600,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part where p_size >= (select min(p_size) from part_null where part_null.p_type = part.p_type) AND p_retailprice <= (select max(p_retailprice) from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2637,9 +2637,9 @@ POSTHOOK: Input: default@part_null 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[91][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 12' is a cross product -Warning: Shuffle Join MERGEJOIN[93][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[91][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 12' is a cross product +Warning: Shuffle Join MERGEJOIN[93][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 13' is a cross product PREHOOK: query: explain select * from part where p_brand <> (select min(p_brand) from part ) AND p_size IN (select (p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_brand <> (select min(p_brand) from part ) AND p_size IN (select (p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 @@ -2653,20 +2653,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE), Reducer 16 (CUSTOM_SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 18 (CUSTOM_SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (CUSTOM_SIMPLE_EDGE) - Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (PARTITION_EDGE), Vertex 16 (PARTITION_EDGE) + Vertex 13 <- Vertex 12 (PARTITION_EDGE), Vertex 18 (PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (PARTITION_EDGE) + Vertex 18 <- Vertex 17 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2684,7 +2684,24 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: part @@ -2699,7 +2716,56 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 15 + Vertex 12 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Vertex 13 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 14 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 15 Map Operator Tree: TableScan alias: part @@ -2719,150 +2785,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string) - outputColumnNames: p_brand - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_brand) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Reducer 13 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2885,7 +2808,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2898,7 +2841,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2920,7 +2863,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 20878 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2938,7 +2881,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2961,7 +2924,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string) + outputColumnNames: p_brand + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_brand) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2973,7 +2956,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3005,9 +3005,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[91][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 12' is a cross product -Warning: Shuffle Join MERGEJOIN[93][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[91][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 12' is a cross product +Warning: Shuffle Join MERGEJOIN[93][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 13' is a cross product PREHOOK: query: select * from part where p_brand <> (select min(p_brand) from part ) AND p_size IN (select (p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3052,13 +3052,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3075,41 +3075,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3134,7 +3100,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3161,7 +3144,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3180,9 +3163,26 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int), _col3 (type: int) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: boolean) - Reducer 7 - Execution mode: llap - Reduce Operator Tree: + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Execution mode: llap + Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial @@ -3231,10 +3231,10 @@ POSTHOOK: Input: default@part 191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[69][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[71][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[69][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[71][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY POSTHOOK: query: explain select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) @@ -3248,19 +3248,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 13 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 15 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3275,7 +3275,19 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Vertex 11 Map Operator Tree: TableScan alias: s1 @@ -3297,83 +3309,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(value) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3392,7 +3328,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3408,7 +3344,31 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3425,7 +3385,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3440,7 +3400,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3470,7 +3430,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3483,7 +3443,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3498,7 +3458,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3523,7 +3483,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3546,6 +3526,26 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(value) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3553,10 +3553,10 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[69][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[71][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[68][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[69][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[71][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 6' is a cross product PREHOOK: query: select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -3575,8 +3575,8 @@ POSTHOOK: Input: default@src 468 4 469 5 489 4 -Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[80][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[80][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part where p_size > (select count(p_name) from part INTERSECT select count(p_brand) from part) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size > (select count(p_name) from part INTERSECT select count(p_brand) from part) @@ -3590,22 +3590,26 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 14 <- Map 13 (CUSTOM_SIMPLE_EDGE) - Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 16 (CONTAINS) - Reducer 17 <- Union 16 (SIMPLE_EDGE) - Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE), Union 16 (CONTAINS) - Reducer 3 <- Reducer 17 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (PARTITION_EDGE) + Vertex 12 <- Union 7 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (PARTITION_EDGE) + Vertex 15 <- Union 16 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) + Vertex 20 <- Union 16 (CONTAINS), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 17 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 16 + Vertex: Union 16 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3620,47 +3624,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string) - outputColumnNames: p_brand - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_brand) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 18 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -3680,27 +3644,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3720,7 +3664,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3741,7 +3685,27 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_name) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3761,7 +3725,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 15 + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3782,7 +3746,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3802,27 +3766,47 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 19 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator + Vertex 18 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string) + outputColumnNames: p_brand + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_brand) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 19 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3837,7 +3821,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3858,7 +3842,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3883,7 +3867,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_name) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3903,7 +3907,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3924,7 +3928,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3951,7 +3955,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3967,10 +3971,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 16 - Vertex: Union 16 - Union 7 - Vertex: Union 7 Stage: Stage-0 Fetch Operator @@ -3978,8 +3978,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[80][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[80][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part where p_size > (select count(p_name) from part INTERSECT select count(p_brand) from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4010,14 +4010,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4034,64 +4034,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4116,7 +4059,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4133,7 +4096,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4156,7 +4119,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8 Data size: 1504 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4175,7 +4138,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 8 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: boolean) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4188,6 +4168,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4223,9 +4223,9 @@ POSTHOOK: Input: default@part 65667 144293 15103 -Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 4' is a cross product PREHOOK: query: explain select * from part_null where p_name NOT LIKE (select min(p_name) from part_null) AND p_brand NOT IN (select p_name from part) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name NOT LIKE (select min(p_name) from part_null) AND p_brand NOT IN (select p_name from part) @@ -4239,17 +4239,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) + Vertex 4 <- Vertex 11 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -4264,7 +4264,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4284,7 +4284,19 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 12 Map Operator Tree: TableScan alias: part @@ -4301,59 +4313,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(p_name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4371,7 +4331,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4386,7 +4346,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 3301 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4408,7 +4368,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 3 Data size: 2535 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4425,7 +4385,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 3 Data size: 2586 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4450,7 +4410,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4473,28 +4453,48 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - - Stage: Stage-0 + Vertex 8 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(p_name) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product +Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part_null where p_name NOT LIKE (select min(p_name) from part_null) AND p_brand NOT IN (select p_name from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4530,8 +4530,8 @@ POSTHOOK: Input: default@part_null 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[78][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[78][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 13' is a cross product PREHOOK: query: explain select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) @@ -4545,21 +4545,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE), Reducer 17 (CUSTOM_SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 15 <- Reducer 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Map 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (PARTITION_EDGE), Vertex 17 (PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -4574,113 +4574,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: p_partkey - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_partkey) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4707,7 +4601,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 17 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4726,7 +4620,22 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 8 Data size: 1685 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: boolean) - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4743,7 +4652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 3400 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4764,7 +4673,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 17 Data size: 3740 Basic stats: COMPLETE Column stats: NONE - Reducer 15 + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4777,7 +4686,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1760 Basic stats: COMPLETE Column stats: NONE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: p_partkey + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(p_partkey) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4790,7 +4719,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4803,7 +4749,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4820,7 +4766,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 5 Data size: 3341 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4844,7 +4790,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 7 Data size: 893 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4869,7 +4815,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4881,7 +4847,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4899,6 +4882,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) + Vertex 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4906,129 +4906,49 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[78][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[78][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[79][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 13' is a cross product PREHOOK: query: select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part PREHOOK: Input: default@part_null -#### A masked pattern was here #### -POSTHOOK: query: select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@part -POSTHOOK: Input: default@part_null -#### A masked pattern was here #### -90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -PREHOOK: query: explain select p.p_partkey, li.l_suppkey -from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey -where li.l_linenumber = 1 and -li.l_orderkey <> (select min(l_orderkey) from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) -PREHOOK: type: QUERY -POSTHOOK: query: explain select p.p_partkey, li.l_suppkey -from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey -where li.l_linenumber = 1 and -li.l_orderkey <> (select min(l_orderkey) from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: l_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 9 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +POSTHOOK: Input: default@part_null +#### A masked pattern was here #### +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +PREHOOK: query: explain select p.p_partkey, li.l_suppkey +from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey +where li.l_linenumber = 1 and +li.l_orderkey <> (select min(l_orderkey) from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) +PREHOOK: type: QUERY +POSTHOOK: query: explain select p.p_partkey, li.l_suppkey +from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey +where li.l_linenumber = 1 and +li.l_orderkey <> (select min(l_orderkey) from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -5048,7 +4968,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5061,7 +4981,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5082,7 +5002,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5095,7 +5015,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 13 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5108,7 +5048,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5125,7 +5065,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 5 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5150,7 +5090,47 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (l_shipmode = 'AIR') (type: boolean) + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5177,7 +5157,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5196,6 +5176,26 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: boolean) + Vertex 9 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: l_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5261,97 +5261,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: l_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -5371,7 +5291,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5384,7 +5304,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5405,7 +5325,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5418,7 +5338,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 13 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5431,7 +5371,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5448,32 +5388,72 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: int) Statistics: Num rows: 5 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col4 (type: int) + 1 _col2 (type: int) + outputColumnNames: _col0, _col1, _col3, _col5, _col6 + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col1 <> CASE WHEN (_col6 is null) THEN (null) ELSE (_col5) END) (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col3 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (l_shipmode = 'AIR') (type: boolean) + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col4 (type: int) - 1 _col2 (type: int) - outputColumnNames: _col0, _col1, _col3, _col5, _col6 - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col1 <> CASE WHEN (_col6 is null) THEN (null) ELSE (_col5) END) (type: boolean) - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5500,7 +5480,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5519,6 +5499,26 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: boolean) + Vertex 9 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: l_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5565,8 +5565,8 @@ POSTHOOK: Input: default@lineitem 155190 7706 175839 874 182052 9607 -Warning: Shuffle Join MERGEJOIN[98][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 9' is a cross product -Warning: Shuffle Join MERGEJOIN[99][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 16' is a cross product +Warning: Shuffle Join MERGEJOIN[98][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 9' is a cross product +Warning: Shuffle Join MERGEJOIN[99][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 16' is a cross product PREHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity > (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY POSTHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity > (select avg(l_quantity) from lineitem where l_partkey = p_partkey) @@ -5580,19 +5580,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 18 (CUSTOM_SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 11 (CUSTOM_SIMPLE_EDGE), Map 8 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (PARTITION_EDGE), Vertex 18 (PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 11 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -5612,7 +5612,20 @@ STAGE PLANS: value expressions: _col1 (type: double), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: part @@ -5627,7 +5640,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 12 Map Operator Tree: TableScan alias: lineitem @@ -5644,94 +5657,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5758,7 +5684,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5777,7 +5703,19 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 11 Data size: 132 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5798,7 +5736,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5811,7 +5749,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 18 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5840,7 +5793,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5855,7 +5808,42 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5876,7 +5864,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5902,7 +5890,19 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5930,8 +5930,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[98][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 9' is a cross product -Warning: Shuffle Join MERGEJOIN[99][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 16' is a cross product +Warning: Shuffle Join MERGEJOIN[98][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 9' is a cross product +Warning: Shuffle Join MERGEJOIN[99][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 16' is a cross product PREHOOK: query: select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity > (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@lineitem @@ -5956,17 +5956,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -5983,24 +5983,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 12 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -6009,49 +5992,15 @@ STAGE PLANS: keys: p_type (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6064,7 +6013,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6077,7 +6043,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6095,7 +6061,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6119,7 +6102,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 13 Data size: 6565 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6140,7 +6123,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6153,7 +6136,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6180,7 +6180,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6374,13 +6374,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: emps @@ -6397,41 +6397,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: deptno (type: int), name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6456,7 +6422,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: deptno (type: int), name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6483,7 +6466,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6502,7 +6485,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: boolean) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6550,13 +6550,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: emps @@ -6573,41 +6573,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: deptno (type: int), name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: deptno (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6632,7 +6598,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: deptno (type: int), name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6659,7 +6642,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6678,7 +6661,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: boolean) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: deptno (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6700,124 +6700,56 @@ STAGE PLANS: PREHOOK: query: select * from emps where name > (select min(name) from depts where depts.deptno=emps.deptno) PREHOOK: type: QUERY -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -#### A masked pattern was here #### -POSTHOOK: query: select * from emps where name > (select min(name) from depts where depts.deptno=emps.deptno) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -#### A masked pattern was here #### -120 Wilma 20 F NULL 1 5 NULL true 2005-09-07 -PREHOOK: query: explain select * from emps where deptno <> (select count(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts where depts.deptno = emps.deptno) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from emps where deptno <> (select count(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts where depts.deptno = emps.deptno) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: empno (type: int), name (type: string), deptno (type: int), gender (type: string), city (type: string), empid (type: int), age (type: int), slacker (type: boolean), manager (type: boolean), joinedat (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: deptno (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: deptno (type: int), name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +#### A masked pattern was here #### +POSTHOOK: query: select * from emps where name > (select min(name) from depts where depts.deptno=emps.deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +#### A masked pattern was here #### +120 Wilma 20 F NULL 1 5 NULL true 2005-09-07 +PREHOOK: query: explain select * from emps where deptno <> (select count(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts where depts.deptno = emps.deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from emps where deptno <> (select count(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts where depts.deptno = emps.deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: emps Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: name (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator + expressions: empno (type: int), name (type: string), deptno (type: int), gender (type: string), city (type: string), empid (type: int), age (type: int), slacker (type: boolean), manager (type: boolean), joinedat (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col0 (type: string) + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: deptno (type: int), name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6844,7 +6776,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6863,7 +6795,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: boolean) - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: deptno (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6876,7 +6825,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 95 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6900,7 +6849,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 5 Data size: 261 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6925,7 +6874,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: deptno (type: int), name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6952,7 +6918,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6971,7 +6937,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: boolean) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6984,6 +6967,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: deptno (type: int), name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -7006,7 +7006,7 @@ POSTHOOK: Input: default@emps 120 Wilma 20 F NULL 1 5 NULL true 2005-09-07 130 Alice 40 F Vancouver 2 NULL false true 2007-01-01 110 John 40 M Vancouver 2 NULL false true 2002-05-03 -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from emps where deptno <> (select sum(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts) PREHOOK: type: QUERY POSTHOOK: query: explain select * from emps where deptno <> (select sum(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts) @@ -7020,16 +7020,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (PARTITION_EDGE) + Vertex 12 <- Vertex 11 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 10 (PARTITION_EDGE), Vertex 12 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: emps @@ -7046,81 +7046,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: name (type: string) - outputColumnNames: name - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: deptno (type: int), name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: deptno (type: int) - outputColumnNames: deptno - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(deptno) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7143,7 +7069,27 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: name (type: string) + outputColumnNames: name + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(name) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7155,7 +7101,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7177,7 +7123,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 261 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7204,7 +7150,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: deptno (type: int), name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7231,7 +7194,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7250,7 +7213,24 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: boolean) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 238 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7263,6 +7243,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: deptno (type: int) + outputColumnNames: deptno + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(deptno) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -7270,7 +7270,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from emps where deptno <> (select count(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts) PREHOOK: type: QUERY PREHOOK: Input: default@depts diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index d96a5a4..25a45e2 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -124,31 +124,31 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 19 <- Map 18 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 32 <- Map 31 (SIMPLE_EDGE) - Reducer 34 <- Map 33 (SIMPLE_EDGE) - Reducer 4 <- Reducer 21 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -169,7 +169,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: a @@ -188,199 +188,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < '11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 27 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 31 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 33 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -401,7 +209,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -419,7 +227,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -436,7 +244,26 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -449,7 +276,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -462,7 +308,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -479,7 +346,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) Statistics: Num rows: 166 Data size: 32204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint), _col5 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -496,7 +363,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) Statistics: Num rows: 166 Data size: 32204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint), _col5 (type: bigint) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -524,7 +391,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -537,7 +404,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -564,7 +450,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -579,7 +465,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -592,7 +497,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -613,7 +537,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -631,7 +555,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -655,7 +579,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -672,7 +596,26 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 32 + Vertex 31 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 32 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -685,7 +628,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 33 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -698,7 +660,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -716,7 +678,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -743,7 +724,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -758,7 +739,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out index 48ea4ce..b7a6368 100644 --- ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out +++ ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out @@ -496,7 +496,7 @@ Keys:val 13.0 1 17.0 1 46.0 1 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 4' is a cross product PREHOOK: query: SELECT * FROM ( diff --git ql/src/test/results/clientpositive/llap/temp_table.q.out ql/src/test/results/clientpositive/llap/temp_table.q.out index e97b3dd..a14b118 100644 --- ql/src/test/results/clientpositive/llap/temp_table.q.out +++ ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -86,7 +86,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -171,10 +171,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: foo @@ -191,7 +191,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -246,12 +246,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: foo @@ -268,24 +270,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: bar - Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -302,8 +287,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 + Vertex 4 + Map Operator Tree: + TableScan + alias: bar + Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out index c83da8b..f9d36da 100644 --- ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out +++ ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out @@ -79,11 +79,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test @@ -103,7 +103,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -112,7 +112,21 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: some inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: test1 @@ -131,20 +145,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_dml.q.out ql/src/test/results/clientpositive/llap/tez_dml.q.out index 786929e..dcb9aca 100644 --- ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -14,11 +14,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -55,7 +55,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -444,7 +444,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tmp_src @@ -865,7 +865,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out index 33350a7..d20b618 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -47,27 +47,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -83,7 +63,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 2166 Data size: 1342920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -97,6 +77,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -178,26 +178,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -216,7 +197,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,6 +212,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -282,12 +282,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -307,26 +307,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -349,7 +330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -363,7 +344,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -377,6 +358,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -434,11 +434,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,27 +458,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -489,7 +469,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator @@ -497,7 +477,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -511,6 +491,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -592,26 +592,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -621,7 +602,7 @@ STAGE PLANS: 0 KEY.reducesinkkey0 (type: int) 1 KEY.reducesinkkey0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -633,7 +614,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -648,6 +629,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -699,12 +699,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -724,26 +724,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -754,7 +735,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -769,7 +750,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -783,7 +764,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 750 Data size: 161305 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -797,6 +778,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out index f434a1e..a92ef33 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out @@ -31,11 +31,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -55,7 +55,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -74,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -93,38 +125,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -222,11 +222,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -246,7 +246,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -265,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -284,38 +316,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -437,7 +437,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -456,7 +488,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -475,38 +507,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out index 2918e68..3bf7561 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out @@ -19,11 +19,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -42,23 +42,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) - sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -74,7 +58,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -98,6 +82,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToInteger(_col0) (type: int) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -126,10 +126,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -148,23 +148,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) - sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -185,7 +169,7 @@ STAGE PLANS: 1 UDFToInteger(_col0) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2 Data size: 620 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 @@ -197,6 +181,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToInteger(_col0) (type: int) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_join_hash.q.out ql/src/test/results/clientpositive/llap/tez_join_hash.q.out index ef4556a..a6687cf 100644 --- ql/src/test/results/clientpositive/llap/tez_join_hash.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_hash.q.out @@ -31,11 +31,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -54,26 +54,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: orc_src - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -92,7 +73,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -107,6 +88,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: orc_src + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -146,13 +146,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 8 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -172,65 +174,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -251,7 +195,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -274,7 +218,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -294,8 +238,64 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out index a7ec895..5baa46c 100644 --- ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out @@ -160,10 +160,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -245,7 +245,7 @@ STAGE PLANS: name: default.ct_events_clean Truncated Path -> Alias: /ct_events_clean [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -262,13 +262,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 4 Data size: 19978 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1144,10 +1144,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1229,7 +1229,7 @@ STAGE PLANS: name: default.ct_events_clean Truncated Path -> Alias: /ct_events_clean [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1246,13 +1246,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 4 Data size: 19978 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/llap/tez_join_tests.q.out ql/src/test/results/clientpositive/llap/tez_join_tests.q.out index 4fa5854..fde7ff8 100644 --- ql/src/test/results/clientpositive/llap/tez_join_tests.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_tests.q.out @@ -13,13 +13,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -36,40 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +52,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,7 +65,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -114,7 +81,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -128,6 +95,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out index b32e990..77602e9 100644 --- ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out +++ ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out @@ -13,13 +13,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -36,40 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +52,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,7 +65,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -114,7 +81,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -128,6 +95,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_nway_join.q.out ql/src/test/results/clientpositive/llap/tez_nway_join.q.out index cee5b9c..20a5970 100644 --- ql/src/test/results/clientpositive/llap/tez_nway_join.q.out +++ ql/src/test/results/clientpositive/llap/tez_nway_join.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -66,8 +66,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 101 Data size: 404 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -80,7 +80,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -99,7 +114,7 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -118,21 +133,6 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -153,11 +153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -177,7 +177,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 50 Data size: 202 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -186,7 +186,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -199,7 +199,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -218,7 +233,7 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -237,21 +252,6 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -294,11 +294,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -314,39 +314,7 @@ STAGE PLANS: Statistics: Num rows: 46 Data size: 184 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -367,7 +335,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -382,6 +350,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -402,11 +402,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -423,7 +423,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 50 Data size: 202 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -432,7 +432,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -445,7 +445,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -461,7 +476,7 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -477,21 +492,6 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_self_join.q.out ql/src/test/results/clientpositive/llap/tez_self_join.q.out index 8ba9761..470a14a 100644 --- ql/src/test/results/clientpositive/llap/tez_self_join.q.out +++ ql/src/test/results/clientpositive/llap/tez_self_join.q.out @@ -38,7 +38,7 @@ POSTHOOK: query: insert into table tez_self_join2 values(1),(2),(3) POSTHOOK: type: QUERY POSTHOOK: Output: default@tez_self_join2 POSTHOOK: Lineage: tez_self_join2.id1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select s.id2, s.id3 from @@ -70,11 +70,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: self1 @@ -92,41 +92,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: self2 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('ab' = id3) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: tez_self_join2 - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: id1 is not null (type: boolean) - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -147,7 +113,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -169,6 +135,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: self2 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('ab' = id3) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: tez_self_join2 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: id1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -176,7 +176,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select s.id2, s.id3 from ( diff --git ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out index d9d2db4..1bb66c6 100644 --- ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out +++ ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out @@ -131,10 +131,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -174,7 +174,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -224,7 +224,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -533,10 +533,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -589,7 +589,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -639,10 +639,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -695,7 +695,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -745,10 +745,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -788,7 +788,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -834,10 +834,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -877,7 +877,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/tez_smb_main.q.out ql/src/test/results/clientpositive/llap/tez_smb_main.q.out index b583bff..43ec596 100644 --- ql/src/test/results/clientpositive/llap/tez_smb_main.q.out +++ ql/src/test/results/clientpositive/llap/tez_smb_main.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -36,7 +36,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -56,24 +74,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -255,11 +255,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -278,7 +278,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -297,7 +297,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -310,7 +310,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -366,11 +366,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -389,7 +389,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -408,7 +408,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -421,7 +421,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -477,11 +477,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -500,7 +500,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -519,7 +519,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -532,7 +532,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -584,11 +584,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -626,7 +626,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -638,7 +638,22 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: c @@ -657,21 +672,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -709,11 +709,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -732,26 +732,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -770,7 +751,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -785,6 +766,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -828,13 +828,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -871,7 +873,41 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 508 Data size: 5388 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 558 Data size: 5926 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: s2 @@ -890,7 +926,7 @@ STAGE PLANS: Statistics: Num rows: 508 Data size: 5388 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -909,42 +945,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 558 Data size: 5926 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -965,11 +965,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -988,7 +988,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1007,7 +1007,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col1 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1020,7 +1020,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1070,11 +1070,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1094,7 +1094,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1114,7 +1114,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1123,7 +1123,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1136,7 +1136,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1155,21 +1170,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1215,12 +1215,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (CUSTOM_EDGE), Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (CO_PARTITION_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1240,7 +1242,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1249,7 +1251,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 558 Data size: 10399 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1262,7 +1264,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -1281,7 +1298,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s2 @@ -1300,7 +1317,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 558 Data size: 10399 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1313,7 +1330,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: b @@ -1337,23 +1354,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1388,12 +1388,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -1411,7 +1411,19 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -1429,19 +1441,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1455,7 +1455,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1466,7 +1466,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/tez_union.q.out ql/src/test/results/clientpositive/llap/tez_union.q.out index 395961e..1903c8a 100644 --- ql/src/test/results/clientpositive/llap/tez_union.q.out +++ ql/src/test/results/clientpositive/llap/tez_union.q.out @@ -17,11 +17,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -41,7 +43,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -52,7 +54,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s3 @@ -71,7 +73,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -89,8 +91,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -171,15 +171,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Map 6 <- Union 7 (CONTAINS) - Map 8 <- Union 7 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) + Vertex 6 <- Union 7 (CONTAINS) + Vertex 8 <- Union 7 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -198,7 +202,41 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 4878 Data size: 39024 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: src @@ -217,7 +255,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: src @@ -236,7 +274,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: src @@ -255,44 +293,6 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 4878 Data size: 39024 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 7 - Vertex: Union 7 Stage: Stage-0 Fetch Operator @@ -351,11 +351,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) - Map 4 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (BROADCAST_EDGE) + Vertex 4 <- Union 3 (CONTAINS), Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -379,7 +381,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src @@ -399,7 +401,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2439 Data size: 424386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -410,7 +412,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src @@ -430,7 +432,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2439 Data size: 424386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -441,8 +443,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -529,13 +529,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 10 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 10 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 7 <- Map 10 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) + Vertex 7 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -555,7 +557,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -567,8 +569,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -581,7 +583,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s8 @@ -610,7 +612,21 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -629,7 +645,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s4 @@ -649,7 +665,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -661,8 +677,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -675,7 +691,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: s5 @@ -694,7 +710,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: s6 @@ -714,7 +730,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -726,8 +742,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -740,7 +756,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: s7 @@ -759,7 +775,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: s1 @@ -788,22 +804,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -971,11 +971,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -995,7 +997,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1006,7 +1008,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s3 @@ -1025,7 +1027,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s4 @@ -1045,7 +1047,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1056,7 +1058,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s5 @@ -1075,8 +1077,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1164,11 +1164,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 3 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1185,7 +1187,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2439 Data size: 868284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1196,7 +1198,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1213,7 +1215,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2439 Data size: 868284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1224,7 +1226,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s @@ -1247,8 +1249,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1275,12 +1275,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (BROADCAST_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1296,7 +1298,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1312,7 +1314,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s @@ -1340,8 +1342,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_union2.q.out ql/src/test/results/clientpositive/llap/tez_union2.q.out index 51d45d2..e1a4bd0 100644 --- ql/src/test/results/clientpositive/llap/tez_union2.q.out +++ ql/src/test/results/clientpositive/llap/tez_union2.q.out @@ -41,14 +41,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS), Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -69,28 +73,22 @@ STAGE PLANS: Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -111,22 +109,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -157,10 +140,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out index 2ca78d7..45f39d3 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out @@ -53,11 +53,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: dummy @@ -80,7 +82,7 @@ STAGE PLANS: name: default.partunion1 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: dummy @@ -103,8 +105,6 @@ STAGE PLANS: name: default.partunion1 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out index 22f24c9..b9b061a 100644 --- ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out @@ -147,16 +147,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS) + Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -182,7 +184,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: v @@ -197,7 +199,50 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (_col1 <= '2014-09-02') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y @@ -223,7 +268,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: z @@ -249,7 +294,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -265,50 +310,7 @@ STAGE PLANS: value expressions: u (type: bigint) Execution mode: llap LLAP IO: unknown - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Filter Operator - predicate: (_col1 <= '2014-09-02') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: bigint) - 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -330,7 +332,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -343,8 +345,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index dc1e778..88c5e97 100644 --- ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -53,14 +53,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -78,73 +80,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s0 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -178,7 +114,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -195,7 +131,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -212,8 +148,72 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: s0 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -952,14 +952,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS), Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s0 @@ -992,58 +994,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1060,7 +1011,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1077,7 +1028,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1111,8 +1080,39 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -1847,14 +1847,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS), Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s0 @@ -1887,58 +1889,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1955,7 +1906,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1972,7 +1923,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2006,8 +1975,39 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -2734,13 +2734,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2758,40 +2760,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2825,7 +2794,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2842,7 +2811,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2859,8 +2828,39 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -3581,13 +3581,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -3605,28 +3607,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3648,7 +3629,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3685,7 +3666,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3702,8 +3683,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out index 5bb596a..20523cb 100644 --- ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -47,27 +47,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -83,7 +63,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 2166 Data size: 1342920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -97,6 +77,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -178,26 +178,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -216,7 +197,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,6 +212,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -282,12 +282,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -307,26 +307,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -349,7 +330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -363,7 +344,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -377,6 +358,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -434,11 +434,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,27 +458,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -489,7 +469,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator @@ -497,7 +477,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -511,6 +491,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -592,26 +592,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -621,7 +602,7 @@ STAGE PLANS: 0 KEY.reducesinkkey0 (type: int) 1 KEY.reducesinkkey0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -633,7 +614,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -648,6 +629,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -699,12 +699,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -724,26 +724,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -754,7 +735,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -769,7 +750,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -783,7 +764,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 750 Data size: 161305 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -797,6 +778,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out index 9c1b3af..e74ea96 100644 --- ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out @@ -31,11 +31,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -55,7 +55,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -74,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -93,38 +125,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -222,11 +222,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -246,7 +246,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -265,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -284,38 +316,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -437,7 +437,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -456,7 +488,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -475,38 +507,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union2.q.out ql/src/test/results/clientpositive/llap/union2.q.out index 8b889af..fdf5bcd 100644 --- ql/src/test/results/clientpositive/llap/union2.q.out +++ ql/src/test/results/clientpositive/llap/union2.q.out @@ -15,12 +15,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -40,7 +42,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -60,23 +77,6 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union3.q.out ql/src/test/results/clientpositive/llap/union3.q.out index 4f07f85..4f20960 100644 --- ql/src/test/results/clientpositive/llap/union3.q.out +++ ql/src/test/results/clientpositive/llap/union3.q.out @@ -41,62 +41,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -112,7 +66,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Limit @@ -127,7 +81,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Limit @@ -142,7 +96,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -156,7 +110,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Limit @@ -171,7 +141,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Limit @@ -186,8 +172,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union4.q.out ql/src/test/results/clientpositive/llap/union4.q.out index b9ca42d..501ab33 100644 --- ql/src/test/results/clientpositive/llap/union4.q.out +++ ql/src/test/results/clientpositive/llap/union4.q.out @@ -29,11 +29,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -51,25 +53,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -93,7 +77,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -117,8 +119,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/union5.q.out ql/src/test/results/clientpositive/llap/union5.q.out index c6a8712..61ecec4 100644 --- ql/src/test/results/clientpositive/llap/union5.q.out +++ ql/src/test/results/clientpositive/llap/union5.q.out @@ -17,12 +17,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -42,27 +44,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -86,7 +68,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -102,7 +84,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -126,8 +128,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union6.q.out ql/src/test/results/clientpositive/llap/union6.q.out index dca14c1..498f0f0 100644 --- ql/src/test/results/clientpositive/llap/union6.q.out +++ ql/src/test/results/clientpositive/llap/union6.q.out @@ -29,11 +29,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 4 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -51,26 +53,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4647 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -90,8 +73,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4647 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/union7.q.out ql/src/test/results/clientpositive/llap/union7.q.out index 61a5ea4..2a0cc3b 100644 --- ql/src/test/results/clientpositive/llap/union7.q.out +++ ql/src/test/results/clientpositive/llap/union7.q.out @@ -17,12 +17,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -42,30 +44,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -89,7 +68,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -105,8 +84,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union8.q.out ql/src/test/results/clientpositive/llap/union8.q.out index 4f3be5f..04317b9 100644 --- ql/src/test/results/clientpositive/llap/union8.q.out +++ ql/src/test/results/clientpositive/llap/union8.q.out @@ -17,12 +17,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -40,7 +42,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -58,7 +60,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -76,8 +78,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union9.q.out ql/src/test/results/clientpositive/llap/union9.q.out index 94e6c02..dfc400e 100644 --- ql/src/test/results/clientpositive/llap/union9.q.out +++ ql/src/test/results/clientpositive/llap/union9.q.out @@ -17,13 +17,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -43,7 +45,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -63,7 +80,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s3 @@ -83,23 +100,6 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out index 81939da..7e25de2 100644 --- ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out +++ ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out @@ -33,14 +33,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 5 (CONTAINS), Vertex 9 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -58,43 +62,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -116,7 +84,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -138,7 +106,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -156,7 +124,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -176,7 +144,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -198,10 +184,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 + Vertex 9 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -273,14 +273,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 5 (CONTAINS), Vertex 9 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -298,43 +302,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -356,7 +324,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -378,7 +346,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -396,7 +364,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -421,7 +389,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -443,10 +429,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 + Vertex 9 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -508,14 +508,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 5 (CONTAINS), Vertex 9 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -533,43 +537,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -591,7 +559,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -613,7 +581,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -631,7 +599,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -651,7 +619,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable12 - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -673,10 +659,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 + Vertex 9 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -746,12 +746,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -772,7 +774,22 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -793,23 +810,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1157,12 +1157,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -1183,25 +1185,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1226,7 +1210,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1248,8 +1250,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1313,14 +1313,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Union 3 (CONTAINS) - Map 8 <- Union 5 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS) + Vertex 8 <- Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1338,49 +1342,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1402,7 +1364,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1420,7 +1382,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1445,10 +1407,48 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1564,59 +1564,107 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 51 <- Union 2 (CONTAINS) - Map 52 <- Union 4 (CONTAINS) - Map 53 <- Union 6 (CONTAINS) - Map 54 <- Union 8 (CONTAINS) - Map 55 <- Union 10 (CONTAINS) - Map 56 <- Union 12 (CONTAINS) - Map 57 <- Union 14 (CONTAINS) - Map 58 <- Union 16 (CONTAINS) - Map 59 <- Union 18 (CONTAINS) - Map 60 <- Union 20 (CONTAINS) - Map 61 <- Union 22 (CONTAINS) - Map 62 <- Union 24 (CONTAINS) - Map 63 <- Union 26 (CONTAINS) - Map 64 <- Union 28 (CONTAINS) - Map 65 <- Union 30 (CONTAINS) - Map 66 <- Union 32 (CONTAINS) - Map 67 <- Union 34 (CONTAINS) - Map 68 <- Union 36 (CONTAINS) - Map 69 <- Union 38 (CONTAINS) - Map 70 <- Union 40 (CONTAINS) - Map 71 <- Union 42 (CONTAINS) - Map 72 <- Union 44 (CONTAINS) - Map 73 <- Union 46 (CONTAINS) - Map 74 <- Union 48 (CONTAINS) - Reducer 11 <- Union 10 (SIMPLE_EDGE), Union 12 (CONTAINS) - Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) - Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) - Reducer 17 <- Union 16 (SIMPLE_EDGE), Union 18 (CONTAINS) - Reducer 19 <- Union 18 (SIMPLE_EDGE), Union 20 (CONTAINS) - Reducer 21 <- Union 20 (SIMPLE_EDGE), Union 22 (CONTAINS) - Reducer 23 <- Union 22 (SIMPLE_EDGE), Union 24 (CONTAINS) - Reducer 25 <- Union 24 (SIMPLE_EDGE), Union 26 (CONTAINS) - Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) - Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 31 <- Union 30 (SIMPLE_EDGE), Union 32 (CONTAINS) - Reducer 33 <- Union 32 (SIMPLE_EDGE), Union 34 (CONTAINS) - Reducer 35 <- Union 34 (SIMPLE_EDGE), Union 36 (CONTAINS) - Reducer 37 <- Union 36 (SIMPLE_EDGE), Union 38 (CONTAINS) - Reducer 39 <- Union 38 (SIMPLE_EDGE), Union 40 (CONTAINS) - Reducer 41 <- Union 40 (SIMPLE_EDGE), Union 42 (CONTAINS) - Reducer 43 <- Union 42 (SIMPLE_EDGE), Union 44 (CONTAINS) - Reducer 45 <- Union 44 (SIMPLE_EDGE), Union 46 (CONTAINS) - Reducer 47 <- Union 46 (SIMPLE_EDGE), Union 48 (CONTAINS) - Reducer 49 <- Union 48 (SIMPLE_EDGE) - Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 50 <- Reducer 49 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 9 <- Union 10 (CONTAINS), Union 8 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 12 (CONTAINS) + Vertex 13 <- Union 12 (SORT_PARTITION_EDGE), Union 14 (CONTAINS) + Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) + Vertex 17 <- Union 16 (SORT_PARTITION_EDGE), Union 18 (CONTAINS) + Vertex 19 <- Union 18 (SORT_PARTITION_EDGE), Union 20 (CONTAINS) + Vertex 21 <- Union 20 (SORT_PARTITION_EDGE), Union 22 (CONTAINS) + Vertex 23 <- Union 22 (SORT_PARTITION_EDGE), Union 24 (CONTAINS) + Vertex 25 <- Union 24 (SORT_PARTITION_EDGE), Union 26 (CONTAINS) + Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) + Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 32 (CONTAINS) + Vertex 33 <- Union 32 (SORT_PARTITION_EDGE), Union 34 (CONTAINS) + Vertex 35 <- Union 34 (SORT_PARTITION_EDGE), Union 36 (CONTAINS) + Vertex 37 <- Union 36 (SORT_PARTITION_EDGE), Union 38 (CONTAINS) + Vertex 39 <- Union 38 (SORT_PARTITION_EDGE), Union 40 (CONTAINS) + Vertex 41 <- Union 40 (SORT_PARTITION_EDGE), Union 42 (CONTAINS) + Vertex 43 <- Union 42 (SORT_PARTITION_EDGE), Union 44 (CONTAINS) + Vertex 45 <- Union 44 (SORT_PARTITION_EDGE), Union 46 (CONTAINS) + Vertex 47 <- Union 46 (SORT_PARTITION_EDGE), Union 48 (CONTAINS) + Vertex 49 <- Union 48 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Union 6 (CONTAINS) + Vertex 50 <- Vertex 49 (PARTITION_EDGE) + Vertex 51 <- Union 2 (CONTAINS) + Vertex 52 <- Union 4 (CONTAINS) + Vertex 53 <- Union 6 (CONTAINS) + Vertex 54 <- Union 8 (CONTAINS) + Vertex 55 <- Union 10 (CONTAINS) + Vertex 56 <- Union 12 (CONTAINS) + Vertex 57 <- Union 14 (CONTAINS) + Vertex 58 <- Union 16 (CONTAINS) + Vertex 59 <- Union 18 (CONTAINS) + Vertex 60 <- Union 20 (CONTAINS) + Vertex 61 <- Union 22 (CONTAINS) + Vertex 62 <- Union 24 (CONTAINS) + Vertex 63 <- Union 26 (CONTAINS) + Vertex 64 <- Union 28 (CONTAINS) + Vertex 65 <- Union 30 (CONTAINS) + Vertex 66 <- Union 32 (CONTAINS) + Vertex 67 <- Union 34 (CONTAINS) + Vertex 68 <- Union 36 (CONTAINS) + Vertex 69 <- Union 38 (CONTAINS) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) + Vertex 70 <- Union 40 (CONTAINS) + Vertex 71 <- Union 42 (CONTAINS) + Vertex 72 <- Union 44 (CONTAINS) + Vertex 73 <- Union 46 (CONTAINS) + Vertex 74 <- Union 48 (CONTAINS) + Vertex 9 <- Union 10 (CONTAINS), Union 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 10 + Vertex: Union 10 + Union 12 + Vertex: Union 12 + Union 14 + Vertex: Union 14 + Union 16 + Vertex: Union 16 + Union 18 + Vertex: Union 18 + Union 2 + Vertex: Union 2 + Union 20 + Vertex: Union 20 + Union 22 + Vertex: Union 22 + Union 24 + Vertex: Union 24 + Union 26 + Vertex: Union 26 + Union 28 + Vertex: Union 28 + Union 30 + Vertex: Union 30 + Union 32 + Vertex: Union 32 + Union 34 + Vertex: Union 34 + Union 36 + Vertex: Union 36 + Union 38 + Vertex: Union 38 + Union 4 + Vertex: Union 4 + Union 40 + Vertex: Union 40 + Union 42 + Vertex: Union 42 + Union 44 + Vertex: Union 44 + Union 46 + Vertex: Union 46 + Union 48 + Vertex: Union 48 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1637,238 +1685,419 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 51 - Map Operator Tree: - TableScan - alias: src + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 13 Execution mode: llap - LLAP IO: no inputs - Map 52 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 15 Execution mode: llap - LLAP IO: no inputs - Map 53 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 17 Execution mode: llap - LLAP IO: no inputs - Map 54 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 19 Execution mode: llap - LLAP IO: no inputs - Map 55 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 21 Execution mode: llap - LLAP IO: no inputs - Map 56 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 23 Execution mode: llap - LLAP IO: no inputs - Map 57 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 25 Execution mode: llap - LLAP IO: no inputs - Map 58 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 27 Execution mode: llap - LLAP IO: no inputs - Map 59 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 29 Execution mode: llap - LLAP IO: no inputs - Map 60 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Execution mode: llap - LLAP IO: no inputs - Map 61 - Map Operator Tree: - TableScan - alias: src + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 31 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 33 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 35 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 37 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 39 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 41 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 43 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 45 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 47 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 49 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 50 Execution mode: llap - LLAP IO: no inputs - Map 62 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 51 Map Operator Tree: TableScan alias: src @@ -1889,7 +2118,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 63 + Vertex 52 Map Operator Tree: TableScan alias: src @@ -1910,7 +2139,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 64 + Vertex 53 Map Operator Tree: TableScan alias: src @@ -1931,7 +2160,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 65 + Vertex 54 Map Operator Tree: TableScan alias: src @@ -1952,7 +2181,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 66 + Vertex 55 Map Operator Tree: TableScan alias: src @@ -1973,7 +2202,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 67 + Vertex 56 Map Operator Tree: TableScan alias: src @@ -1994,7 +2223,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 68 + Vertex 57 Map Operator Tree: TableScan alias: src @@ -2015,7 +2244,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 69 + Vertex 58 Map Operator Tree: TableScan alias: src @@ -2036,7 +2265,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 70 + Vertex 59 Map Operator Tree: TableScan alias: src @@ -2057,7 +2286,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 71 + Vertex 60 Map Operator Tree: TableScan alias: src @@ -2078,7 +2307,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 72 + Vertex 61 Map Operator Tree: TableScan alias: src @@ -2099,7 +2328,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 73 + Vertex 62 Map Operator Tree: TableScan alias: src @@ -2120,7 +2349,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 74 + Vertex 63 Map Operator Tree: TableScan alias: src @@ -2141,331 +2370,133 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 15 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 25 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 27 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 29 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 31 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 33 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Vertex 64 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 35 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 65 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 37 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 66 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 39 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 67 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 41 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 68 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 43 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 69 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 45 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2483,95 +2514,112 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 47 + Vertex 70 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 71 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 49 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator + LLAP IO: no inputs + Vertex 72 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 5 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 73 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 50 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + LLAP IO: no inputs + Vertex 74 + Map Operator Tree: + TableScan + alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2589,54 +2637,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union 10 - Vertex: Union 10 - Union 12 - Vertex: Union 12 - Union 14 - Vertex: Union 14 - Union 16 - Vertex: Union 16 - Union 18 - Vertex: Union 18 - Union 2 - Vertex: Union 2 - Union 20 - Vertex: Union 20 - Union 22 - Vertex: Union 22 - Union 24 - Vertex: Union 24 - Union 26 - Vertex: Union 26 - Union 28 - Vertex: Union 28 - Union 30 - Vertex: Union 30 - Union 32 - Vertex: Union 32 - Union 34 - Vertex: Union 34 - Union 36 - Vertex: Union 36 - Union 38 - Vertex: Union 38 - Union 4 - Vertex: Union 4 - Union 40 - Vertex: Union 40 - Union 42 - Vertex: Union 42 - Union 44 - Vertex: Union 44 - Union 46 - Vertex: Union 46 - Union 48 - Vertex: Union 48 - Union 6 - Vertex: Union 6 - Union 8 - Vertex: Union 8 Stage: Stage-0 Fetch Operator @@ -2754,13 +2754,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2778,28 +2780,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2821,7 +2802,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2858,7 +2839,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2875,8 +2856,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -3609,12 +3609,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -3632,28 +3634,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3675,7 +3656,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3703,8 +3684,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest218 - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -4437,12 +4437,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -4460,28 +4462,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4503,7 +4484,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4537,8 +4518,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest219 - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -5245,13 +5245,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -5272,28 +5274,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5312,7 +5293,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5327,8 +5308,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5378,16 +5378,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 11 <- Union 9 (CONTAINS) - Map 6 <- Union 3 (CONTAINS) - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 9 (CONTAINS) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Union 9 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) + Vertex 8 <- Union 9 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -5403,34 +5407,24 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: s4 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Vertex 11 Map Operator Tree: TableScan - alias: s2 + alias: s4 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(key) < 10.0) (type: boolean) @@ -5451,37 +5445,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5503,7 +5467,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5517,7 +5481,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5535,7 +5499,47 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5557,10 +5561,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -5638,18 +5638,26 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 10 <- Union 2 (CONTAINS) - Map 11 <- Union 4 (CONTAINS) - Map 12 <- Union 6 (CONTAINS) - Map 13 <- Union 8 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 9 <- Union 8 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 10 <- Union 2 (CONTAINS) + Vertex 11 <- Union 4 (CONTAINS) + Vertex 12 <- Union 6 (CONTAINS) + Vertex 13 <- Union 8 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Union 6 (CONTAINS) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) + Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -5670,7 +5678,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 37720 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: src @@ -5691,7 +5699,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 37720 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 11 Map Operator Tree: TableScan alias: src @@ -5712,7 +5720,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 37720 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 12 Map Operator Tree: TableScan alias: src_thrift @@ -5733,7 +5741,7 @@ STAGE PLANS: Statistics: Num rows: 108 Data size: 19872 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: no inputs - Map 13 + Vertex 13 Map Operator Tree: TableScan alias: src_thrift @@ -5754,7 +5762,7 @@ STAGE PLANS: Statistics: Num rows: 59 Data size: 10856 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5772,7 +5780,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 37720 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5790,7 +5798,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 108 Data size: 19872 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5808,7 +5816,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 59 Data size: 10856 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5829,14 +5837,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 6 - Vertex: Union 6 - Union 8 - Vertex: Union 8 Stage: Stage-0 Fetch Operator @@ -6497,12 +6497,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: dst_union22_delta @@ -6583,7 +6585,46 @@ STAGE PLANS: name: default.dst_union22_delta Truncated Path -> Alias: /dst_union22_delta/ds=1 [dst_union22_delta] - Map 4 + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 174 Data size: 20274 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ + Statistics: Num rows: 174 Data size: 20274 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: a @@ -6605,13 +6646,13 @@ STAGE PLANS: filter predicates: 0 {(_col2 = '1')} 1 - Estimated key counts: Map 5 => 55 + Estimated key counts: Vertex 5 => 55 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col4, _col5 input vertices: - 1 Map 5 + 1 Vertex 5 Position of Big Table: 0 Statistics: Num rows: 182 Data size: 33598 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -6685,7 +6726,7 @@ STAGE PLANS: name: default.dst_union22 Truncated Path -> Alias: /dst_union22/ds=1 [a] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: dst_union22_delta @@ -6760,49 +6801,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22_delta name: default.dst_union22_delta - Truncated Path -> Alias: - /dst_union22_delta/ds=1 [dst_union22_delta] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 174 Data size: 20274 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=2/ - Statistics: Num rows: 174 Data size: 20274 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Union 2 - Vertex: Union 2 + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] Stage: Stage-2 Dependency Collection @@ -7234,12 +7234,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -7265,7 +7267,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -7286,23 +7303,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -7716,16 +7716,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 8 <- Union 2 (CONTAINS) - Map 9 <- Union 4 (CONTAINS) - Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 11 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Union 6 (CONTAINS) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 2 (CONTAINS) + Vertex 9 <- Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -7805,7 +7811,7 @@ STAGE PLANS: name: default.src2 Truncated Path -> Alias: /src2 [src2] - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: src5 @@ -7887,7 +7893,105 @@ STAGE PLANS: name: default.src5 Truncated Path -> Alias: /src5 [src5] - Map 8 + Vertex 11 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Vertex 5 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Vertex 7 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 8 Map Operator Tree: TableScan alias: src3 @@ -7967,7 +8071,7 @@ STAGE PLANS: name: default.src3 Truncated Path -> Alias: /src3 [src3] - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: src4 @@ -7989,168 +8093,64 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Execution mode: llap - LLAP IO: no inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src4 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4 - name: default.src4 - Truncated Path -> Alias: - /src4 [src4] - Reducer 11 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 7 + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + LLAP IO: no inputs + Path -> Alias: #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE + Path -> Partition: #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 6 - Vertex: Union 6 + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [src4] Stage: Stage-0 Fetch Operator @@ -8223,14 +8223,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Map 7 <- Map 8 (BROADCAST_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS) + Vertex 7 <- Union 4 (CONTAINS), Vertex 8 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -8310,7 +8314,60 @@ STAGE PLANS: name: default.src2 Truncated Path -> Alias: /src2 [src2] - Map 6 + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Vertex 5 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 6 Map Operator Tree: TableScan alias: src3 @@ -8390,7 +8447,7 @@ STAGE PLANS: name: default.src3 Truncated Path -> Alias: /src3 [src3] - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: a @@ -8407,13 +8464,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 8 => 103 + Estimated key counts: Vertex 8 => 103 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 8 + 1 Vertex 8 Position of Big Table: 0 Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -8486,7 +8543,7 @@ STAGE PLANS: name: default.src4 Truncated Path -> Alias: /src4 [a] - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: b @@ -8562,63 +8619,6 @@ STAGE PLANS: name: default.src5 Truncated Path -> Alias: /src5 [b] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -8685,15 +8685,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Map 7 <- Map 9 (BROADCAST_EDGE) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS) + Vertex 7 <- Vertex 9 (BROADCAST_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -8773,7 +8777,60 @@ STAGE PLANS: name: default.src2 Truncated Path -> Alias: /src2 [src2] - Map 6 + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Vertex 5 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 6 Map Operator Tree: TableScan alias: src3 @@ -8853,7 +8910,7 @@ STAGE PLANS: name: default.src3 Truncated Path -> Alias: /src3 [src3] - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: a @@ -8870,13 +8927,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 9 => 103 + Estimated key counts: Vertex 9 => 103 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 9 + 1 Vertex 9 Position of Big Table: 0 Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -8947,7 +9004,30 @@ STAGE PLANS: name: default.src4 Truncated Path -> Alias: /src4 [a] - Map 9 + Vertex 8 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: true + Vertex 9 Map Operator Tree: TableScan alias: b @@ -9022,86 +9102,6 @@ STAGE PLANS: name: default.src5 Truncated Path -> Alias: /src5 [b] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 8 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE - tag: -1 - auto parallelism: true - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -9213,14 +9213,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS), Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9241,49 +9245,7 @@ STAGE PLANS: Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9309,7 +9271,28 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmp_unionall - Reducer 6 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9340,10 +9323,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 375 Data size: 66750 Basic stats: COMPLETE Column stats: PARTIAL - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -9419,12 +9419,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9440,7 +9442,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) @@ -9454,7 +9456,32 @@ STAGE PLANS: Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 775 Data size: 8233 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -9469,7 +9496,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: srcpart @@ -9523,33 +9550,6 @@ STAGE PLANS: Statistics: Num rows: 1550 Data size: 16467 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 775 Data size: 8233 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 387 Data size: 4111 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -10683,12 +10683,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Union 3 (CONTAINS) - Map 5 <- Union 3 (CONTAINS) - Reducer 4 <- Map 1 (BROADCAST_EDGE), Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 1 (BROADCAST_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10700,41 +10702,17 @@ STAGE PLANS: expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 2 - Map Operator Tree: - TableScan - alias: dim_pho - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) = 97.0) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 Map Operator Tree: TableScan - alias: jackson_sev_add + alias: dim_pho Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(key) = 97.0) (type: boolean) @@ -10755,7 +10733,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -10771,7 +10749,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -10784,8 +10762,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: jackson_sev_add + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 97.0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -10852,14 +10852,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS), Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -10880,30 +10884,27 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) + Vertex 3 Execution mode: llap - LLAP IO: no inputs - Map 8 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Vertex 4 Map Operator Tree: TableScan alias: src @@ -10926,27 +10927,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -10965,7 +10946,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -10987,7 +10968,30 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11006,10 +11010,6 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 6 - Vertex: Union 6 Stage: Stage-2 Dependency Collection @@ -11130,14 +11130,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS), Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -11158,7 +11162,27 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 + Vertex 4 Map Operator Tree: TableScan alias: src @@ -11179,7 +11203,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 7 Map Operator Tree: TableScan alias: src @@ -11200,48 +11242,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union29 - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 Stage: Stage-2 Dependency Collection @@ -11360,61 +11360,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 14 <- Map 13 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (PARTITION_EDGE) + Vertex 12 <- Union 5 (CONTAINS), Vertex 11 (PARTITION_EDGE) + Vertex 14 <- Union 7 (CONTAINS), Vertex 13 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) + Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -11429,7 +11390,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Limit @@ -11449,7 +11410,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Limit @@ -11469,7 +11445,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Limit @@ -11489,7 +11480,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Limit @@ -11509,7 +11500,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11527,7 +11518,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11545,7 +11536,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11560,12 +11551,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -11691,16 +11691,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 12 <- Union 4 (CONTAINS) - Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 8 (CONTAINS) - Reducer 9 <- Union 2 (CONTAINS), Union 8 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 11 <- Union 8 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 4 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 8 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 2 (CONTAINS), Union 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 8 + Vertex: Union 8 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -11721,7 +11727,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: src @@ -11744,7 +11750,26 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 12 Map Operator Tree: TableScan alias: src @@ -11765,49 +11790,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11825,7 +11808,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11845,7 +11828,30 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union30 - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11864,7 +11870,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -11886,12 +11892,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 8 - Vertex: Union 8 Stage: Stage-2 Dependency Collection @@ -12064,13 +12064,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -12091,28 +12093,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12158,7 +12139,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12179,8 +12160,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t4 - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -12324,13 +12324,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -12353,30 +12355,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12395,7 +12374,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12409,7 +12388,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Forward @@ -12450,7 +12429,30 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t6 - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12469,8 +12471,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Union 3 - Vertex: Union 3 Stage: Stage-3 Dependency Collection @@ -12656,13 +12656,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -12685,28 +12687,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), cnt (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12725,7 +12706,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -12742,7 +12723,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Forward @@ -12783,8 +12764,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.t8 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), cnt (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -12938,12 +12938,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -12964,7 +12966,22 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: t2 @@ -12985,23 +13002,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -13052,12 +13052,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13077,7 +13079,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double) @@ -13095,7 +13097,22 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -13114,7 +13131,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: t2 @@ -13135,23 +13152,6 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -13202,12 +13202,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -13228,7 +13230,22 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -13248,7 +13265,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double) @@ -13266,7 +13283,7 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: b @@ -13285,23 +13302,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -13352,12 +13352,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13377,7 +13379,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: string) @@ -13395,7 +13397,22 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -13414,7 +13431,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: t2 @@ -13435,23 +13452,6 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -13502,12 +13502,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -13528,7 +13530,22 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -13548,7 +13565,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(CAST( _col1 AS varchar(20))) (type: string) @@ -13566,7 +13583,7 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 147 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: b @@ -13585,23 +13602,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -13672,12 +13672,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -13701,7 +13703,23 @@ STAGE PLANS: Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + Vertex 4 Map Operator Tree: TableScan alias: src @@ -13724,23 +13742,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -13763,8 +13765,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection @@ -13844,12 +13844,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -13872,31 +13874,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 0.0) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -13919,7 +13897,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -13935,8 +13913,30 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 227 Data size: 61517 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -14108,14 +14108,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Union 6 (CONTAINS) - Map 8 <- Union 6 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS) + Vertex 7 <- Union 2 (CONTAINS), Union 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 6 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: src10_1 @@ -14135,7 +14139,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) @@ -14149,7 +14153,22 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src10_2 @@ -14168,7 +14187,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: src10_3 @@ -14189,7 +14208,25 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE + Vertex 8 Map Operator Tree: TableScan alias: src10_4 @@ -14210,43 +14247,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE - Union 2 - Vertex: Union 2 - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -14309,14 +14309,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 7 (CONTAINS) - Map 9 <- Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 8 <- Union 3 (CONTAINS), Union 7 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 7 (CONTAINS) + Vertex 8 <- Union 3 (CONTAINS), Union 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 7 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src10_1 @@ -14336,7 +14340,43 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: src10_2 @@ -14355,7 +14395,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: src10_3 @@ -14376,7 +14416,25 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE + Vertex 9 Map Operator Tree: TableScan alias: src10_4 @@ -14397,64 +14455,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 103 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 218 Basic stats: COMPLETE Column stats: NONE - Union 3 - Vertex: Union 3 - Union 7 - Vertex: Union 7 Stage: Stage-0 Fetch Operator @@ -14533,12 +14533,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -14556,25 +14558,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14596,7 +14580,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14616,7 +14600,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14638,8 +14640,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -14702,12 +14702,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -14725,25 +14727,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14765,7 +14749,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14790,7 +14774,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14812,8 +14814,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -14874,12 +14874,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -14897,28 +14899,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14940,7 +14921,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -14956,8 +14937,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -15043,12 +15043,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -15066,28 +15068,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -15109,7 +15090,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -15134,8 +15115,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -15193,14 +15193,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Map 7 <- Union 4 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS) + Vertex 7 <- Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -15221,7 +15225,40 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: s2 @@ -15242,7 +15279,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: s3 @@ -15263,43 +15300,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -15647,15 +15647,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Map 8 <- Union 4 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) + Vertex 8 <- Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -15676,49 +15680,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -15736,7 +15698,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -15755,7 +15717,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -15770,10 +15732,48 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union_remove_26.q.out ql/src/test/results/clientpositive/llap/union_remove_26.q.out index 18b9aa5..72a383d 100644 --- ql/src/test/results/clientpositive/llap/union_remove_26.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_26.q.out @@ -357,12 +357,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inputtbl1 @@ -557,7 +557,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: inputtbl2 @@ -577,7 +592,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: inputtbl3 @@ -597,37 +627,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -720,7 +720,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: inputtbl2 @@ -740,7 +755,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: inputtbl3 @@ -760,37 +790,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,8 +805,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union_stats.q.out ql/src/test/results/clientpositive/llap/union_stats.q.out index 1f3dc82..1a58a38 100644 --- ql/src/test/results/clientpositive/llap/union_stats.q.out +++ ql/src/test/results/clientpositive/llap/union_stats.q.out @@ -14,11 +14,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -102,7 +104,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -186,8 +188,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/union_top_level.q.out ql/src/test/results/clientpositive/llap/union_top_level.q.out index de23f48..bdbc193 100644 --- ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -21,12 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -48,7 +50,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), 0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -70,7 +93,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: src @@ -92,49 +136,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), 0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), 1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -155,8 +157,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -208,13 +208,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -234,7 +236,45 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: s1 @@ -253,7 +293,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: s2 @@ -273,26 +313,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -313,7 +334,7 @@ STAGE PLANS: Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 3 + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -330,46 +351,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Reducer 8 + Vertex 9 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -437,12 +437,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -464,51 +466,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -530,7 +488,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -552,7 +532,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -574,8 +576,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -670,12 +670,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -697,51 +699,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -763,7 +721,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -785,7 +765,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -807,8 +809,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -890,12 +890,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -917,51 +919,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -983,7 +941,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1005,7 +985,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1027,8 +1029,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out index 8482ed9..80516df 100644 --- ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out +++ ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out @@ -117,7 +117,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -185,7 +185,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -253,7 +253,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -321,7 +321,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -389,7 +389,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -457,7 +457,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -515,7 +515,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -609,7 +609,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -667,7 +667,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -761,7 +761,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -821,10 +821,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -847,7 +847,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -898,10 +898,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -924,7 +924,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out index b093ded..b707c87 100644 --- ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out +++ ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -139,7 +139,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out index fdd3d1b..9cddc77 100644 --- ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out +++ ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out @@ -38,18 +38,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_14] Group By Operator [GBY_13] (rows=1 width=188) Output:["_col0","_col1"],aggregations:["max(VALUE._col0)","max(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap - PARTITION_ONLY_SHUFFLE [RS_12] + <-Vertex 1 [PARTITION_EDGE] vectorized, llap + PARTITION [RS_12] Group By Operator [GBY_11] (rows=1 width=188) Output:["_col0","_col1"],aggregations:["max(dt)","max(greg_dt)"] Select Operator [SEL_10] (rows=3 width=102) diff --git ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index bc37741..8619214 100644 --- ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -51,18 +51,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=11 width=93) @@ -118,26 +118,26 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 vectorized, llap + Vertex 3 vectorized, llap File Output Operator [FS_31] Group By Operator [GBY_30] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap - PARTITION_ONLY_SHUFFLE [RS_29] + <-Vertex 2 [PARTITION_EDGE] vectorized, llap + PARTITION [RS_29] Group By Operator [GBY_28] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Select Operator [SEL_27] (rows=5 width=93) Group By Operator [GBY_26] (rows=5 width=93) Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Group By Operator [GBY_10] (rows=11 width=93) Output:["_col0"],keys:_col0 @@ -218,27 +218,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap + Vertex 3 llap File Output Operator [FS_32] Select Operator [SEL_31] (rows=5 width=102) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_49] (rows=5 width=102) Conds:RS_51._col0=RS_53._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 2 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_51] + <-Vertex 2 [SORT_PARTITION_EDGE] vectorized, llap + PARTITION_SORT [RS_51] PartitionCols:_col0 Group By Operator [GBY_50] (rows=5 width=93) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Group By Operator [GBY_10] (rows=11 width=93) Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 @@ -256,13 +256,13 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=10 width=93) default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Reducer 6 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_53] + <-Vertex 6 [SORT_PARTITION_EDGE] vectorized, llap + PARTITION_SORT [RS_53] PartitionCols:_col0 Group By Operator [GBY_52] (rows=5 width=93) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_25] PartitionCols:_col0 Group By Operator [GBY_24] (rows=11 width=93) Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 @@ -342,18 +342,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=3 width=102) @@ -417,18 +417,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=1 width=102) @@ -516,18 +516,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=1 width=102) @@ -603,18 +603,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=3 width=102) @@ -668,25 +668,25 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 vectorized, llap + Vertex 3 vectorized, llap File Output Operator [FS_29] Group By Operator [GBY_28] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 2 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_21] (rows=11 width=93) Conds:RS_24._col0=RS_27._col0(Inner) - <-Map 1 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_24] + <-Vertex 1 [SORT_PARTITION_EDGE] vectorized, llap + PARTITION_SORT [RS_24] PartitionCols:_col0 Select Operator [SEL_23] (rows=10 width=93) Output:["_col0"] @@ -694,8 +694,8 @@ Stage-0 predicate:(key + 1) is not null TableScan [TS_0] (rows=10 width=93) default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_27] + <-Vertex 4 [SORT_PARTITION_EDGE] vectorized, llap + PARTITION_SORT [RS_27] PartitionCols:_col0 Select Operator [SEL_26] (rows=10 width=93) Output:["_col0"] @@ -736,18 +736,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=3 width=102) @@ -803,18 +803,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_32] Group By Operator [GBY_31] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_15] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_15] Group By Operator [GBY_14] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_28] (rows=6 width=102) @@ -892,18 +892,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_22] Group By Operator [GBY_21] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] + <-Vertex 1 [PARTITION_EDGE] llap + PARTITION [RS_11] Group By Operator [GBY_10] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_19] (rows=1 width=102) @@ -992,7 +992,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Map 1 llap + Vertex 1 llap File Output Operator [FS_11] table:{"name:":"default.dest1"} Select Operator [SEL_10] (rows=11 width=93) @@ -1141,7 +1141,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-4 Stats-Aggr Operator @@ -1151,14 +1151,14 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_27] table:{"name:":"default.dest2"} Select Operator [SEL_26] (rows=5 width=93) Output:["_col0","_col1"] Group By Operator [GBY_25] (rows=5 width=93) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap + <-Vertex 1 [SORT_PARTITION_EDGE] llap File Output Operator [FS_11] table:{"name:":"default.dest1"} Merge Join Operator [MERGEJOIN_23] (rows=11 width=93) @@ -1175,7 +1175,7 @@ Stage-4 predicate:key is not null TableScan [TS_0] (rows=10 width=93) default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - SHUFFLE [RS_14] + PARTITION_SORT [RS_14] PartitionCols:_col0 Group By Operator [GBY_13] (rows=11 width=93) Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 diff --git ql/src/test/results/clientpositive/llap/vector_between_columns.q.out ql/src/test/results/clientpositive/llap/vector_between_columns.q.out index 1ed359f..98e8804 100644 --- ql/src/test/results/clientpositive/llap/vector_between_columns.q.out +++ ql/src/test/results/clientpositive/llap/vector_between_columns.q.out @@ -60,7 +60,7 @@ POSTHOOK: Output: default@TINT POSTHOOK: Lineage: tint.cint SIMPLE [(tint_txt)tint_txt.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: tint.rnum SIMPLE [(tint_txt)tint_txt.FieldSchema(name:rnum, type:int, comment:null), ] tint_txt.rnum tint_txt.cint -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint PREHOOK: type: QUERY @@ -77,10 +77,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tint @@ -97,7 +97,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col1 (type: int), _col3 (type: smallint), CASE WHEN (_col1 BETWEEN _col3 AND _col3) THEN ('Ok') ELSE ('NoOk') END (type: string) @@ -112,7 +112,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tsint @@ -134,7 +134,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint PREHOOK: type: QUERY PREHOOK: Input: default@tint @@ -171,7 +171,7 @@ tint.rnum tsint.rnum tint.cint tsint.csint between_col 4 2 10 0 NoOk 4 3 10 1 NoOk 4 4 10 10 Ok -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY @@ -188,10 +188,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tint @@ -208,7 +208,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col1 BETWEEN _col3 AND _col3 (type: boolean) @@ -226,7 +226,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tsint @@ -248,7 +248,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY PREHOOK: Input: default@tint diff --git ql/src/test/results/clientpositive/llap/vector_between_in.q.out ql/src/test/results/clientpositive/llap/vector_between_in.q.out index 74384a4..215bd6f 100644 --- ql/src/test/results/clientpositive/llap/vector_between_in.q.out +++ ql/src/test/results/clientpositive/llap/vector_between_in.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -46,7 +46,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -80,10 +80,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -139,10 +139,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -160,7 +160,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -194,10 +194,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -218,7 +218,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -274,7 +274,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 274112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -308,10 +308,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -329,7 +329,7 @@ STAGE PLANS: Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -363,10 +363,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -384,7 +384,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 274112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -418,10 +418,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -442,7 +442,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -722,11 +722,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -749,7 +749,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -763,7 +763,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -797,11 +797,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -824,7 +824,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -838,7 +838,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -872,11 +872,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -899,7 +899,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -913,7 +913,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -947,11 +947,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -974,7 +974,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -988,7 +988,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out index 7de04a7..ca20c11 100644 --- ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out +++ ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out @@ -112,11 +112,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -136,7 +136,7 @@ STAGE PLANS: 1 _col10 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 110 Data size: 32601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18,_col19,_col20,_col21) (type: int) @@ -153,7 +153,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -173,21 +188,6 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -225,10 +225,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: hundredorc @@ -251,7 +251,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -333,10 +333,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -356,7 +356,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 110 Data size: 32601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: binary), _col3 (type: binary) @@ -371,7 +371,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 diff --git ql/src/test/results/clientpositive/llap/vector_bucket.q.out ql/src/test/results/clientpositive/llap/vector_bucket.q.out index 0573abf..a68f849 100644 --- ql/src/test/results/clientpositive/llap/vector_bucket.q.out +++ ql/src/test/results/clientpositive/llap/vector_bucket.q.out @@ -23,10 +23,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: values__tmp__table__1 @@ -42,7 +42,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out index 9a2456f..a8ce732 100644 --- ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out +++ ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out @@ -118,11 +118,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -161,7 +161,7 @@ STAGE PLANS: Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: decimal(14,4)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_char_2.q.out ql/src/test/results/clientpositive/llap/vector_char_2.q.out index bdcf7d8..631ebec 100644 --- ql/src/test/results/clientpositive/llap/vector_char_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_2.q.out @@ -68,11 +68,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -96,7 +96,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -111,7 +111,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -198,11 +198,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -226,7 +226,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -241,7 +241,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_char_4.q.out ql/src/test/results/clientpositive/llap/vector_char_4.q.out index 6d55ab0..cedcd8a 100644 --- ql/src/test/results/clientpositive/llap/vector_char_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_4.q.out @@ -138,7 +138,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc diff --git ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out index f2a386e..737c4de 100644 --- ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out @@ -137,11 +137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -161,7 +161,7 @@ STAGE PLANS: 1 _col1 (type: char(10)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -170,7 +170,21 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: char(10)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: char(10)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -190,20 +204,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: char(10)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -237,11 +237,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -261,7 +261,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -281,7 +281,7 @@ STAGE PLANS: 1 _col1 (type: char(20)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -290,7 +290,7 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: char(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -339,11 +339,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -363,7 +363,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -372,7 +372,21 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -392,20 +406,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_char_simple.q.out ql/src/test/results/clientpositive/llap/vector_char_simple.q.out index b17bf57..dd7f1ab 100644 --- ql/src/test/results/clientpositive/llap/vector_char_simple.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_simple.q.out @@ -64,10 +64,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -84,7 +84,7 @@ STAGE PLANS: value expressions: _col1 (type: char(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -165,10 +165,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -185,7 +185,7 @@ STAGE PLANS: value expressions: _col1 (type: char(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -265,10 +265,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -287,7 +287,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out index 18f45ff..de3b155 100644 --- ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out @@ -35,10 +35,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -61,7 +61,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,7 +121,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -181,10 +181,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -207,7 +207,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -267,7 +267,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc diff --git ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out index 5eb4035..5c319b4 100644 --- ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out +++ ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: m @@ -66,7 +66,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), CASE WHEN ((COALESCE(_col2,5) > 1)) THEN (_col2) ELSE (null) END (type: bigint) @@ -81,7 +81,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: n diff --git ql/src/test/results/clientpositive/llap/vector_complex_all.q.out ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index a7ae10e..fdd627e 100644 --- ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -150,7 +150,7 @@ b str two line1 four line2 six line3 -Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_0]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_0]] in Stage 'Vertex 2' is a cross product PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT orc_create_staging.*, src1.key FROM orc_create_staging cross join src src1 cross join orc_create_staging spam1 cross join orc_create_staging spam2 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_complex_join.q.out ql/src/test/results/clientpositive/llap/vector_complex_join.q.out index 89507ee..ab88540 100644 --- ql/src/test/results/clientpositive/llap/vector_complex_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_complex_join.q.out @@ -33,10 +33,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -56,7 +56,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 10090 Data size: 2540021 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -67,7 +67,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test @@ -156,10 +156,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2b @@ -175,7 +175,7 @@ STAGE PLANS: 1 a[1] (type: int) outputColumnNames: _col0, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col4 (type: array) @@ -190,7 +190,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2a diff --git ql/src/test/results/clientpositive/llap/vector_count.q.out ql/src/test/results/clientpositive/llap/vector_count.q.out index 9ef5c2b..fa54abe 100644 --- ql/src/test/results/clientpositive/llap/vector_count.q.out +++ ql/src/test/results/clientpositive/llap/vector_count.q.out @@ -56,10 +56,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -82,7 +82,7 @@ STAGE PLANS: value expressions: _col5 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -130,10 +130,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -155,7 +155,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -199,10 +199,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -219,7 +219,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -267,10 +267,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -285,7 +285,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out index 3456d45..2175a6d 100644 --- ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out +++ ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out @@ -1240,11 +1240,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: web_sales @@ -1265,7 +1265,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 3520000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1282,7 +1282,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_data_types.q.out ql/src/test/results/clientpositive/llap/vector_data_types.q.out index 045f536..93a923c 100644 --- ql/src/test/results/clientpositive/llap/vector_data_types.q.out +++ ql/src/test/results/clientpositive/llap/vector_data_types.q.out @@ -108,10 +108,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -128,7 +128,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -204,10 +204,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -224,7 +224,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_date_1.q.out ql/src/test/results/clientpositive/llap/vector_date_1.q.out index 8fbbb0a..dd29f3e 100644 --- ql/src/test/results/clientpositive/llap/vector_date_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_date_1.q.out @@ -83,10 +83,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -102,7 +102,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -193,10 +193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -212,7 +212,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -303,10 +303,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -322,7 +322,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -413,10 +413,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -432,7 +432,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -521,10 +521,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -543,7 +543,7 @@ STAGE PLANS: value expressions: _col1 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -633,10 +633,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -654,7 +654,7 @@ STAGE PLANS: value expressions: _col1 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out index b3ff557..e8d5454 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out @@ -47,10 +47,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -65,7 +65,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -110,10 +110,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -128,7 +128,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -173,10 +173,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -191,7 +191,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -254,7 +254,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -299,10 +299,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -317,7 +317,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -362,10 +362,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -380,7 +380,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -425,10 +425,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -443,7 +443,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -488,10 +488,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -506,7 +506,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -551,10 +551,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -569,7 +569,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out index 5576078..4540f14 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out @@ -48,10 +48,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal @@ -66,7 +66,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out index 144356c..f10f82a 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out @@ -36,10 +36,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -54,7 +54,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -99,10 +99,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -117,7 +117,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -162,10 +162,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -180,7 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -225,10 +225,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -243,7 +243,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -306,7 +306,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -351,10 +351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -369,7 +369,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -414,10 +414,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -432,7 +432,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -495,7 +495,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -551,10 +551,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -569,7 +569,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -614,10 +614,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -632,7 +632,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -677,10 +677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -695,7 +695,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -740,10 +740,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -758,7 +758,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -803,10 +803,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -821,7 +821,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -866,10 +866,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -884,7 +884,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -929,10 +929,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -947,7 +947,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -992,10 +992,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1010,7 +1010,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1055,10 +1055,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1070,7 +1070,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1115,10 +1115,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1130,7 +1130,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1175,10 +1175,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1190,7 +1190,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1235,10 +1235,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1250,7 +1250,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1286,10 +1286,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1301,7 +1301,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1346,10 +1346,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1361,7 +1361,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1406,10 +1406,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1421,7 +1421,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1466,10 +1466,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1481,7 +1481,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1526,10 +1526,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1541,7 +1541,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1586,10 +1586,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1601,7 +1601,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1646,10 +1646,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1661,7 +1661,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out index a8d1e05..08ffe2e 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_vgby @@ -69,7 +69,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)), _col9 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -148,10 +148,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_vgby @@ -174,7 +174,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out index eddb4dc..d254af0 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out @@ -87,10 +87,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -110,7 +110,7 @@ STAGE PLANS: 1 _col0 (type: decimal(6,2)) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -121,7 +121,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 diff --git ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out index c16f605..7cf3d3b 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out @@ -581,7 +581,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: decimal(30,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out index 8e7cd63..d3b65c2 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_txt @@ -62,7 +62,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -107,10 +107,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_txt @@ -126,7 +126,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -197,10 +197,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_rc @@ -216,7 +216,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -261,10 +261,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_rc @@ -280,7 +280,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -351,10 +351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_orc @@ -370,7 +370,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -415,10 +415,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_orc @@ -434,7 +434,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out index 29ec85b..0d14fca 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out @@ -47,10 +47,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_1_orc @@ -66,7 +66,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(22,1)), _col3 (type: decimal(23,2)), _col4 (type: decimal(24,3)), _col5 (type: decimal(21,0)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(21,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -160,10 +160,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_2_orc @@ -179,7 +179,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(22,1)), _col3 (type: decimal(23,2)), _col4 (type: decimal(24,3)), _col5 (type: decimal(25,4)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(22,1)), _col13 (type: decimal(23,2)), _col14 (type: decimal(24,3)), _col15 (type: decimal(25,4)), _col16 (type: decimal(21,0)), _col17 (type: decimal(21,0)), _col18 (type: decimal(21,0)), _col19 (type: decimal(21,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -300,10 +300,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_3_orc @@ -319,7 +319,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(21,0)), _col3 (type: decimal(21,0)), _col4 (type: decimal(21,0)), _col5 (type: decimal(21,0)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(21,0)), _col13 (type: decimal(21,0)), _col14 (type: decimal(21,0)), _col15 (type: decimal(21,0)), _col16 (type: decimal(21,0)), _col17 (type: decimal(22,1)), _col18 (type: decimal(23,2)), _col19 (type: decimal(24,3)), _col20 (type: decimal(25,4)), _col21 (type: decimal(26,5)), _col22 (type: decimal(27,6)), _col23 (type: decimal(28,7)), _col24 (type: decimal(29,8)), _col25 (type: decimal(30,9)), _col26 (type: decimal(31,10)), _col27 (type: decimal(32,11)), _col28 (type: decimal(33,12)), _col29 (type: decimal(34,13)), _col31 (type: decimal(35,14)), _col32 (type: decimal(36,15)), _col33 (type: decimal(37,16)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -429,10 +429,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_4_orc @@ -448,7 +448,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(30,9)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out index 1be938e..9e13f1a 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -145,7 +145,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -229,7 +229,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -313,7 +313,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -397,7 +397,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -481,7 +481,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -565,7 +565,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -649,7 +649,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -733,7 +733,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -817,7 +817,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -889,7 +889,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -973,7 +973,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1057,7 +1057,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1141,7 +1141,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1191,7 +1191,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1241,7 +1241,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1324,7 +1324,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1397,7 +1397,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1470,7 +1470,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1554,7 +1554,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1638,11 +1638,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1665,7 +1665,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(30,10)), _col2 (type: bigint), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1683,7 +1683,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(38,18)), _col2 (type: decimal(24,14)), _col3 (type: decimal(30,10)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1742,7 +1742,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1891,7 +1891,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1975,7 +1975,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2059,7 +2059,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2143,7 +2143,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2227,7 +2227,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2311,10 +2311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2337,7 +2337,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2398,10 +2398,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2424,7 +2424,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2508,7 +2508,7 @@ STAGE PLANS: value expressions: _col0 (type: array) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2575,7 +2575,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2642,7 +2642,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2709,7 +2709,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out index 5ea9f4d..3fda0f9 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf2 @@ -127,7 +127,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf2 diff --git ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out index 620b99e..a9bf4a5 100644 --- ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -140,7 +140,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby4.q.out ql/src/test/results/clientpositive/llap/vector_groupby4.q.out index 9ecfa56..58be235 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby4.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby4.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcorc @@ -57,7 +57,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -70,7 +70,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby6.q.out ql/src/test/results/clientpositive/llap/vector_groupby6.q.out index 4c6e038..a860944 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby6.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby6.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcorc @@ -57,7 +57,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -70,7 +70,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out index 742cebc..9cf9141 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -142,7 +142,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out index a0a3393..91dfdc5 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from src @@ -16,36 +16,36 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Reducer 4 (BROADCAST_EDGE), Reducer 6 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap + Vertex 2 vectorized, llap File Output Operator [FS_37] Select Operator [SEL_36] (rows=500 width=178) Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_22] Select Operator [SEL_21] (rows=500 width=178) Output:["_col0","_col1"] Filter Operator [FIL_20] (rows=500 width=198) predicate:((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) Map Join Operator [MAPJOIN_28] (rows=500 width=198) Conds:MAPJOIN_27._col0=RS_35._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 6 [BROADCAST_EDGE] vectorized, llap + <-Vertex 6 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_35] PartitionCols:_col0 Select Operator [SEL_34] (rows=205 width=91) Output:["_col0","_col1"] Group By Operator [GBY_33] (rows=205 width=87) Output:["_col0"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Group By Operator [GBY_10] (rows=205 width=87) Output:["_col0"],keys:key @@ -53,12 +53,12 @@ Stage-0 default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] <-Map Join Operator [MAPJOIN_27] (rows=500 width=194) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [BROADCAST_EDGE] vectorized, llap + <-Vertex 4 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_32] Group By Operator [GBY_31] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] + <-Vertex 3 [PARTITION_EDGE] llap + PARTITION [RS_5] Group By Operator [GBY_4] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count()","count(key)"] Select Operator [SEL_3] (rows=500 width=87) @@ -70,7 +70,7 @@ Stage-0 TableScan [TS_0] (rows=500 width=178) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] -Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select * from src where not key in diff --git ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out index c4bcbab..8c295c4 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -261,7 +261,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -357,11 +357,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -382,7 +382,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -404,7 +404,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -566,11 +566,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -593,7 +593,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -617,7 +617,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -779,11 +779,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -806,7 +806,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -830,7 +830,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out index 77a0695..2cb8f4d 100644 --- ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out +++ ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out @@ -142,10 +142,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -166,7 +166,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -227,10 +227,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -251,7 +251,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -315,10 +315,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -339,7 +339,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out index c3c5773..edfbe9f 100644 --- ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out +++ ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out @@ -162,7 +162,7 @@ POSTHOOK: Lineage: customer_demographics.cd_education_status SIMPLE [(customer_d POSTHOOK: Lineage: customer_demographics.cd_gender SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_gender, type:string, comment:null), ] POSTHOOK: Lineage: customer_demographics.cd_marital_status SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_marital_status, type:string, comment:null), ] POSTHOOK: Lineage: customer_demographics.cd_purchase_estimate SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_purchase_estimate, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Vertex 2' is a cross product PREHOOK: query: explain select count(1) from customer_demographics,store_sales where ((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'M') or @@ -197,7 +197,7 @@ STAGE PLANS: value expressions: cd_demo_sk (type: int), cd_marital_status (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: store_sales @@ -210,7 +210,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 200000 Data size: 92055200 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col0 = _col16) and (_col2 = 'M')) or ((_col0 = _col16) and (_col2 = 'U'))) (type: boolean) @@ -228,7 +228,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -250,7 +250,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Vertex 2' is a cross product PREHOOK: query: select count(1) from customer_demographics,store_sales where ((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'M') or (customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'U')) diff --git ql/src/test/results/clientpositive/llap/vector_inner_join.q.out ql/src/test/results/clientpositive/llap/vector_inner_join.q.out index d50123d..dbc9e6d 100644 --- ql/src/test/results/clientpositive/llap/vector_inner_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_inner_join.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -66,7 +66,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) @@ -81,7 +81,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -133,10 +133,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -156,7 +156,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -167,7 +167,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -256,10 +256,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -279,7 +279,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: int) @@ -294,7 +294,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -347,10 +347,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -370,7 +370,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -390,7 +390,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -434,10 +434,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -457,7 +457,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), (_col3 * 2) (type: int), (_col0 * 5) (type: int), _col1 (type: string) @@ -472,7 +472,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -525,10 +525,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -548,7 +548,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col0 (type: int) @@ -563,7 +563,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -616,10 +616,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -639,7 +639,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col2 (type: string), _col1 (type: string) @@ -654,7 +654,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -707,10 +707,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -730,7 +730,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -750,7 +750,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col2 (type: int) @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -821,7 +821,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -841,7 +841,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: string) diff --git ql/src/test/results/clientpositive/llap/vector_interval_1.q.out ql/src/test/results/clientpositive/llap/vector_interval_1.q.out index 207e9bb..fab8131 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_1.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -80,7 +80,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_year_month), _col4 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -150,10 +150,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -169,7 +169,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -247,10 +247,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -266,7 +266,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col5 (type: interval_day_time), _col6 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -356,10 +356,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -375,7 +375,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date), _col7 (type: timestamp), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -496,7 +496,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp), _col7 (type: timestamp), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -580,10 +580,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -599,7 +599,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -665,10 +665,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -684,7 +684,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -756,10 +756,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -775,7 +775,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time), _col5 (type: interval_day_time), _col6 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_interval_2.q.out ql/src/test/results/clientpositive/llap/vector_interval_2.q.out index d84737c..b9f783e 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_2.q.out @@ -115,10 +115,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -134,7 +134,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean), _col18 (type: boolean), _col19 (type: boolean), _col20 (type: boolean), _col21 (type: boolean), _col22 (type: boolean), _col23 (type: boolean), _col24 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -284,10 +284,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -303,7 +303,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -453,10 +453,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -472,7 +472,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean), _col18 (type: boolean), _col19 (type: boolean), _col20 (type: boolean), _col21 (type: boolean), _col22 (type: boolean), _col23 (type: boolean), _col24 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -622,10 +622,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -641,7 +641,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -777,10 +777,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -798,7 +798,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -931,10 +931,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -952,7 +952,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1075,10 +1075,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1096,7 +1096,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1219,10 +1219,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1240,7 +1240,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1373,10 +1373,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1394,7 +1394,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1527,10 +1527,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1548,7 +1548,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out index 391c775..15eb309 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out @@ -69,10 +69,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -88,7 +88,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -214,10 +214,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -233,7 +233,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -359,10 +359,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -378,7 +378,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -502,10 +502,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -519,7 +519,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -598,10 +598,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -617,7 +617,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -745,10 +745,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -764,7 +764,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -892,10 +892,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -911,7 +911,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1033,7 +1033,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 diff --git ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out index 0bc0e4c..2d5d8a1 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out @@ -189,10 +189,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab_a_1korc @@ -212,7 +212,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: interval_day_time) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1100 Data size: 506290 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time) @@ -227,7 +227,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: vectortab_b_1korc diff --git ql/src/test/results/clientpositive/llap/vector_join30.q.out ql/src/test/results/clientpositive/llap/vector_join30.q.out index d7e5f58..9a3c86b 100644 --- ql/src/test/results/clientpositive/llap/vector_join30.q.out +++ ql/src/test/results/clientpositive/llap/vector_join30.q.out @@ -35,12 +35,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -58,26 +58,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -92,7 +73,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -103,7 +84,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -118,7 +99,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -182,12 +182,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -202,23 +202,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -233,7 +217,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -244,7 +228,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -259,7 +243,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -323,12 +323,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -343,7 +343,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Vertex 3 Map Operator Tree: TableScan alias: orcsrc @@ -359,19 +371,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -386,7 +386,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -397,7 +397,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -469,13 +469,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -493,45 +493,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -548,8 +510,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 - 2 Reducer 7 + 1 Vertex 5 + 2 Vertex 7 Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -560,7 +522,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -575,7 +537,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -588,7 +569,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -663,14 +663,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -685,39 +685,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -729,7 +697,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -751,7 +719,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -766,7 +734,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -779,7 +763,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -854,14 +854,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -876,39 +876,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -920,7 +888,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -942,7 +910,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -957,7 +925,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -970,7 +954,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1045,14 +1045,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -1067,39 +1067,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1111,7 +1079,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1133,7 +1101,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1148,7 +1116,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1161,7 +1145,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1236,14 +1236,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orcsrc @@ -1258,39 +1258,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1302,7 +1270,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1324,7 +1292,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1339,7 +1307,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1352,7 +1336,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_join_filters.q.out ql/src/test/results/clientpositive/llap/vector_join_filters.q.out index 1a492b6..2f3e2f6 100644 --- ql/src/test/results/clientpositive/llap/vector_join_filters.q.out +++ ql/src/test/results/clientpositive/llap/vector_join_filters.q.out @@ -26,7 +26,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -36,7 +36,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -46,7 +46,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out index 3497164..689c279 100644 --- ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out +++ ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out @@ -26,7 +26,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -36,7 +36,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -46,7 +46,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out index e8dfc30..6f61753 100644 --- ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -46,7 +46,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 104600 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 3418417 Data size: 27347336 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -68,7 +68,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -84,7 +99,7 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -100,21 +115,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out index 57d63b3..8475089 100644 --- ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out +++ ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out @@ -91,10 +91,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -114,7 +114,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -129,7 +129,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -181,10 +181,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -204,7 +204,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -219,7 +219,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -271,10 +271,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -294,7 +294,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -309,7 +309,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -361,10 +361,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -384,7 +384,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -399,7 +399,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -451,10 +451,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -474,7 +474,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -489,7 +489,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -541,10 +541,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -564,7 +564,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -579,7 +579,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 diff --git ql/src/test/results/clientpositive/llap/vector_leftsemi_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_leftsemi_mapjoin.q.out index 6300316..7c00c08 100644 --- ql/src/test/results/clientpositive/llap/vector_leftsemi_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_leftsemi_mapjoin.q.out @@ -141,11 +141,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -161,7 +161,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -169,7 +169,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -193,20 +207,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -243,11 +243,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -263,7 +263,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -271,7 +271,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -295,20 +309,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -347,11 +347,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -367,7 +367,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -375,7 +375,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -399,20 +413,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -443,11 +443,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -463,7 +463,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -475,7 +475,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -499,20 +513,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -554,11 +554,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -574,7 +574,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -582,7 +582,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -606,20 +620,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -653,11 +653,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -681,7 +681,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -697,7 +697,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -709,7 +709,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -756,11 +756,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -784,7 +784,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -800,7 +800,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -812,7 +812,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -856,11 +856,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -884,7 +884,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -900,7 +900,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -908,7 +908,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -957,11 +957,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -977,7 +977,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -985,7 +985,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1009,20 +1023,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1072,11 +1072,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1092,7 +1092,7 @@ STAGE PLANS: 1 (2 * _col0) (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -1100,7 +1100,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1124,20 +1138,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1172,11 +1172,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1194,8 +1194,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) @@ -1208,7 +1208,21 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: string) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1224,7 +1238,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1248,20 +1262,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1308,11 +1308,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1328,7 +1328,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -1336,7 +1336,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1360,20 +1374,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1418,11 +1418,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1440,8 +1440,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1449,7 +1449,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1473,7 +1487,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1497,20 +1511,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1557,11 +1557,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1576,8 +1576,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -1585,7 +1585,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1597,7 +1611,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1618,20 +1632,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1690,11 +1690,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1706,7 +1706,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1718,7 +1749,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1739,37 +1770,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1831,11 +1831,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1847,7 +1847,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1868,7 +1899,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1880,37 +1911,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1972,11 +1972,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1988,7 +1988,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2009,7 +2040,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2021,37 +2052,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2115,11 +2115,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2131,7 +2131,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2152,7 +2183,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2164,37 +2195,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2269,11 +2269,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2289,7 +2289,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -2299,7 +2299,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -2307,7 +2307,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2331,7 +2345,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -2343,20 +2357,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2424,10 +2424,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2447,7 +2447,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2458,7 +2458,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -2512,11 +2512,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2532,7 +2532,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -2540,7 +2540,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2564,20 +2578,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2614,11 +2614,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2634,7 +2634,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -2642,7 +2642,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2666,20 +2680,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2718,11 +2718,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2738,7 +2738,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -2746,7 +2746,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2770,20 +2784,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2814,11 +2814,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2834,7 +2834,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -2846,7 +2846,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2870,20 +2884,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2925,11 +2925,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2945,7 +2945,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -2953,7 +2953,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2977,20 +2991,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3024,11 +3024,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -3052,7 +3052,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -3068,7 +3068,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -3080,7 +3080,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3127,11 +3127,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -3155,7 +3155,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -3171,7 +3171,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -3183,7 +3183,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3227,11 +3227,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -3255,7 +3255,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -3271,7 +3271,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -3279,7 +3279,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3328,11 +3328,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3348,7 +3348,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -3356,7 +3356,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3380,20 +3394,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3443,11 +3443,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3463,7 +3463,7 @@ STAGE PLANS: 1 (2 * _col0) (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -3471,7 +3471,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3495,20 +3509,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3543,11 +3543,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3565,8 +3565,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) @@ -3579,7 +3579,21 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: string) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3595,7 +3609,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -3619,20 +3633,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3679,11 +3679,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3699,7 +3699,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -3707,7 +3707,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3731,20 +3745,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3789,11 +3789,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3811,8 +3811,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -3820,7 +3820,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3844,7 +3858,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -3868,20 +3882,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3928,11 +3928,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3947,8 +3947,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -3956,7 +3956,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3968,7 +3982,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -3989,20 +4003,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4061,11 +4061,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4077,7 +4077,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -4089,7 +4120,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -4110,37 +4141,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4202,11 +4202,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4218,7 +4218,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -4239,7 +4270,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -4251,37 +4282,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4343,11 +4343,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4359,7 +4359,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -4380,7 +4411,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -4392,37 +4423,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4486,11 +4486,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4502,40 +4502,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4552,7 +4519,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4566,6 +4533,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -4640,11 +4640,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4660,7 +4660,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -4670,7 +4670,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -4678,7 +4678,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -4702,7 +4716,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -4714,20 +4728,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4795,10 +4795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4818,7 +4818,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4829,7 +4829,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -4883,11 +4883,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -4903,7 +4903,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -4911,7 +4911,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -4935,20 +4949,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4985,11 +4985,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5005,7 +5005,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -5013,7 +5013,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -5037,20 +5051,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5089,11 +5089,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5109,7 +5109,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -5117,7 +5117,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -5141,20 +5155,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5185,11 +5185,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5205,7 +5205,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -5217,7 +5217,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -5241,20 +5255,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5296,11 +5296,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5316,7 +5316,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -5324,7 +5324,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -5348,20 +5362,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5395,11 +5395,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -5423,7 +5423,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -5439,7 +5439,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -5451,7 +5451,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -5498,11 +5498,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -5526,7 +5526,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -5542,7 +5542,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -5554,7 +5554,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -5598,11 +5598,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -5626,7 +5626,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -5642,7 +5642,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -5650,7 +5650,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -5699,11 +5699,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5719,7 +5719,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -5727,7 +5727,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -5751,20 +5765,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5814,11 +5814,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5834,7 +5834,7 @@ STAGE PLANS: 1 (2 * _col0) (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -5842,7 +5842,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -5866,20 +5880,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5914,11 +5914,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -5936,8 +5936,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) @@ -5950,13 +5950,27 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) @@ -5966,7 +5980,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -5990,20 +6004,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6050,11 +6050,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6070,7 +6070,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -6078,7 +6078,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -6102,20 +6116,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6160,11 +6160,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6182,8 +6182,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -6191,7 +6191,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -6215,7 +6229,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -6239,20 +6253,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6299,11 +6299,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6318,8 +6318,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -6327,7 +6327,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -6339,7 +6353,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -6360,20 +6374,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6432,11 +6432,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6448,7 +6448,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -6460,7 +6491,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -6481,37 +6512,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6573,11 +6573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6589,7 +6589,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -6610,7 +6641,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -6622,37 +6653,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6714,11 +6714,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6730,7 +6730,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -6751,7 +6782,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -6763,37 +6794,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6857,11 +6857,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -6873,7 +6873,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -6894,7 +6925,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -6906,37 +6937,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7011,11 +7011,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7031,7 +7031,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -7041,7 +7041,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -7049,7 +7049,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -7073,7 +7087,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -7085,20 +7099,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7166,10 +7166,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7189,7 +7189,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7200,7 +7200,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -7254,11 +7254,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7274,7 +7274,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -7282,7 +7282,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -7306,20 +7320,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7356,11 +7356,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7376,7 +7376,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -7384,7 +7384,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -7408,20 +7422,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7460,11 +7460,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7480,7 +7480,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -7488,7 +7488,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -7512,20 +7526,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7556,11 +7556,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7576,7 +7576,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -7588,7 +7588,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -7612,20 +7626,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7667,11 +7667,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -7687,7 +7687,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -7695,7 +7695,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -7719,20 +7733,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7766,11 +7766,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -7794,7 +7794,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -7810,7 +7810,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -7822,7 +7822,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -7869,11 +7869,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -7897,7 +7897,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -7913,7 +7913,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -7925,7 +7925,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -7969,11 +7969,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7997,7 +7997,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -8013,7 +8013,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -8021,7 +8021,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -8070,11 +8070,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8090,7 +8090,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -8098,7 +8098,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -8122,20 +8136,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8185,11 +8185,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8205,7 +8205,7 @@ STAGE PLANS: 1 (2 * _col0) (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -8213,7 +8213,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -8237,20 +8251,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8285,11 +8285,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8307,8 +8307,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) @@ -8321,7 +8321,21 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -8337,7 +8351,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -8361,20 +8375,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8421,11 +8421,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8441,7 +8441,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -8449,7 +8449,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -8473,20 +8487,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8531,11 +8531,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8553,8 +8553,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -8562,7 +8562,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -8586,7 +8600,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -8610,20 +8624,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8670,11 +8670,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8689,8 +8689,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -8698,7 +8698,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -8710,7 +8724,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -8731,20 +8745,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8803,11 +8803,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8819,7 +8819,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -8831,7 +8862,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -8852,37 +8883,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -8944,11 +8944,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -8960,7 +8960,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -8981,7 +9012,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -8993,37 +9024,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9085,11 +9085,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9101,7 +9101,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -9122,7 +9153,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -9134,37 +9165,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9228,11 +9228,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9244,7 +9244,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -9265,7 +9296,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -9277,37 +9308,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9382,11 +9382,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9402,7 +9402,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -9412,7 +9412,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -9420,7 +9420,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -9444,7 +9458,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -9456,20 +9470,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9537,10 +9537,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9560,7 +9560,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -9571,7 +9571,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -9625,11 +9625,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9645,7 +9645,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -9653,7 +9653,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -9677,20 +9691,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9727,11 +9727,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9747,7 +9747,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -9755,7 +9755,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -9779,20 +9793,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9831,11 +9831,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9851,7 +9851,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -9859,7 +9859,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -9883,20 +9897,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -9927,11 +9927,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -9947,7 +9947,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -9959,7 +9959,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -9983,20 +9997,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -10038,11 +10038,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10058,7 +10058,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -10066,15 +10066,29 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((value < 'val_10') and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE - Select Operator + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value < 'val_10') and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE + Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE @@ -10090,20 +10104,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -10137,11 +10137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -10165,7 +10165,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -10181,7 +10181,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -10193,7 +10193,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -10240,11 +10240,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -10268,7 +10268,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -10284,7 +10284,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -10296,7 +10296,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -10340,11 +10340,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -10368,7 +10368,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -10384,7 +10384,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -10392,7 +10392,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -10441,11 +10441,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10461,7 +10461,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -10469,7 +10469,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -10493,20 +10507,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -10556,11 +10556,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10576,7 +10576,7 @@ STAGE PLANS: 1 (2 * _col0) (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -10584,7 +10584,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -10608,20 +10622,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -10656,11 +10656,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10678,8 +10678,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) @@ -10692,7 +10692,21 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -10708,7 +10722,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -10732,20 +10746,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -10792,11 +10792,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10812,7 +10812,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -10820,7 +10820,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -10844,20 +10858,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -10902,11 +10902,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -10924,8 +10924,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -10933,7 +10933,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -10957,7 +10971,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -10981,20 +10995,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -11041,11 +11041,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11060,8 +11060,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -11069,7 +11069,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -11081,7 +11095,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -11102,20 +11116,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -11174,11 +11174,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11190,7 +11190,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -11202,7 +11233,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -11223,37 +11254,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -11315,11 +11315,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11331,40 +11331,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -11381,7 +11348,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -11395,6 +11362,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -11456,11 +11456,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11472,7 +11472,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -11493,7 +11524,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -11505,37 +11536,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -11599,11 +11599,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11615,7 +11615,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -11636,7 +11667,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -11648,37 +11679,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -11753,11 +11753,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11773,7 +11773,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -11783,7 +11783,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -11791,7 +11791,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -11815,7 +11829,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -11827,20 +11841,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -11908,10 +11908,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -11931,7 +11931,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -11942,7 +11942,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -11996,11 +11996,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12016,7 +12016,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -12024,7 +12024,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -12048,20 +12062,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -12098,11 +12098,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12118,7 +12118,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -12126,7 +12126,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -12150,20 +12164,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -12202,11 +12202,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12222,7 +12222,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -12230,15 +12230,29 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator expressions: key (type: int) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -12254,20 +12268,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -12298,11 +12298,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12318,7 +12318,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -12330,7 +12330,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -12354,20 +12368,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -12409,11 +12409,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12429,7 +12429,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -12437,7 +12437,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -12461,20 +12475,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -12508,11 +12508,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -12536,7 +12536,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -12552,7 +12552,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -12564,7 +12564,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -12611,11 +12611,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -12639,7 +12639,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -12655,7 +12655,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -12667,7 +12667,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -12711,11 +12711,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -12739,7 +12739,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -12755,7 +12755,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 1 + 1 Vertex 1 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -12763,7 +12763,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -12812,11 +12812,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12832,7 +12832,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -12840,7 +12840,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -12864,20 +12878,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -12927,11 +12927,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -12947,7 +12947,7 @@ STAGE PLANS: 1 (2 * _col0) (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -12955,7 +12955,21 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -12979,20 +12993,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13027,11 +13027,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13049,8 +13049,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) @@ -13063,7 +13063,21 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -13079,7 +13093,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -13103,20 +13117,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13163,11 +13163,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13183,7 +13183,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) @@ -13191,7 +13191,21 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -13215,20 +13229,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13273,11 +13273,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13295,8 +13295,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -13304,7 +13304,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -13328,7 +13342,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -13352,20 +13366,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13412,11 +13412,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13431,8 +13431,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -13440,7 +13440,21 @@ STAGE PLANS: Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -13452,7 +13466,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -13473,20 +13487,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13545,11 +13545,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13561,7 +13561,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -13573,7 +13604,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -13594,37 +13625,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13686,11 +13686,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13702,7 +13702,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -13723,7 +13754,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -13735,37 +13766,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13827,11 +13827,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13843,7 +13843,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -13864,7 +13895,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -13876,37 +13907,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -13970,11 +13970,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -13986,7 +13986,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -14007,7 +14038,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -14019,37 +14050,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -14124,11 +14124,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -14144,7 +14144,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -14154,7 +14154,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -14162,7 +14162,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -14186,7 +14200,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -14198,20 +14212,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2475 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -14279,10 +14279,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -14302,7 +14302,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -14313,7 +14313,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b diff --git ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out index 1960c0c..5438873 100644 --- ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out +++ ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out @@ -241,13 +241,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -267,46 +267,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: unknown - Map 6 - Map Operator Tree: - TableScan - alias: store - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: s_store_sk (type: int), s_state (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: date_dim - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: d_date_sk is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -323,7 +284,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -351,7 +312,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -366,7 +327,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -383,6 +344,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: store + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: s_store_sk (type: int), s_state (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: date_dim + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: d_date_sk is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: d_date_sk (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out index 469c702..a7fde7c 100644 --- ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out @@ -90,7 +90,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git ql/src/test/results/clientpositive/llap/vector_null_projection.q.out ql/src/test/results/clientpositive/llap/vector_null_projection.q.out index a4a36e0..a0e4409 100644 --- ql/src/test/results/clientpositive/llap/vector_null_projection.q.out +++ ql/src/test/results/clientpositive/llap/vector_null_projection.q.out @@ -43,7 +43,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -92,12 +92,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -118,7 +120,26 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: void) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -139,27 +160,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: void) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_nullsafe_join.q.out ql/src/test/results/clientpositive/llap/vector_nullsafe_join.q.out index fba6f18..06d191b 100644 --- ql/src/test/results/clientpositive/llap/vector_nullsafe_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_nullsafe_join.q.out @@ -39,10 +39,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -56,7 +56,7 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int) @@ -71,7 +71,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -123,10 +123,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -144,8 +144,8 @@ STAGE PLANS: 2 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -160,7 +160,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -176,7 +176,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -222,10 +222,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -241,8 +241,8 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -257,7 +257,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -270,7 +270,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -340,10 +340,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -362,8 +362,8 @@ STAGE PLANS: nullSafes: [true, false] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -378,7 +378,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -393,7 +393,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -438,10 +438,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -457,8 +457,8 @@ STAGE PLANS: nullSafes: [true, true] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -473,7 +473,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -485,7 +485,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -628,10 +628,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -645,7 +645,7 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int) @@ -660,7 +660,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -712,10 +712,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -733,8 +733,8 @@ STAGE PLANS: 2 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -749,7 +749,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -765,7 +765,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -811,10 +811,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -830,8 +830,8 @@ STAGE PLANS: nullSafes: [true] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -846,7 +846,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -859,7 +859,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -929,10 +929,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -951,8 +951,8 @@ STAGE PLANS: nullSafes: [true, false] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -967,7 +967,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -982,7 +982,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1027,10 +1027,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1046,8 +1046,8 @@ STAGE PLANS: nullSafes: [true, true] outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) @@ -1062,7 +1062,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1074,7 +1074,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c diff --git ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out index c89eb11..b812114 100644 --- ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out +++ ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out @@ -114,10 +114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2k_orc @@ -193,10 +193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2k_orc diff --git ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out index 503cf5b..a590f9d 100644 --- ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out +++ ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out @@ -116,11 +116,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -143,7 +143,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -157,7 +157,7 @@ STAGE PLANS: sort order: - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out index 3f6aca2..6da48e0 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out @@ -73,10 +73,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -93,7 +93,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 598 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -104,7 +104,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -159,10 +159,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -179,7 +179,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -196,7 +196,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 598 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out index c7cadf1..84b87df 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out @@ -235,10 +235,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -255,7 +255,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 32 Data size: 19648 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -266,7 +266,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: cd @@ -344,10 +344,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -364,7 +364,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 112 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -375,7 +375,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: hd @@ -544,11 +544,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -565,7 +565,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -575,7 +575,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 240 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col0) @@ -588,7 +588,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -604,7 +619,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -620,21 +635,6 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out index a58ce8e..029c210 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out @@ -251,11 +251,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -272,7 +272,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 57 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -282,7 +282,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 162 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col1) @@ -295,7 +295,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -311,7 +326,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -327,21 +342,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out index a34cb8d..2b6b9f7 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out @@ -251,11 +251,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -272,7 +272,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -281,7 +281,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 80 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -294,7 +294,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -310,7 +325,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -326,21 +341,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 979 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -396,11 +396,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -417,7 +417,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 21 Data size: 1869 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -426,7 +426,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 42 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -439,7 +439,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -455,7 +470,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -471,21 +486,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 979 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -541,11 +541,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -562,7 +562,7 @@ STAGE PLANS: 1 _col0 (type: bigint), _col1 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 20 Data size: 1860 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -571,7 +571,7 @@ STAGE PLANS: 0 _col0 (type: int), _col2 (type: string) 1 _col0 (type: int), _col1 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -584,7 +584,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -600,7 +615,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 2120 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -616,21 +631,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1023 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out index 0fc7021..73363dc 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out @@ -265,10 +265,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -285,7 +285,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 81 Data size: 49734 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -296,7 +296,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: cd @@ -409,10 +409,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -429,7 +429,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 450 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -440,7 +440,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: hd @@ -914,11 +914,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -935,7 +935,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 81 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -944,7 +944,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1215 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -957,7 +957,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -973,7 +988,7 @@ STAGE PLANS: Statistics: Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -989,21 +1004,6 @@ STAGE PLANS: Statistics: Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out index 6866862..8bc7a19 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out @@ -105,7 +105,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -118,7 +118,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: st @@ -134,21 +149,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -221,7 +221,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -234,7 +234,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -250,21 +265,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -337,7 +337,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -350,7 +350,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -366,21 +381,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -453,7 +453,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -466,7 +466,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -482,21 +497,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -572,7 +572,7 @@ STAGE PLANS: 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint) @@ -581,39 +581,7 @@ STAGE PLANS: Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: sm - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col0 pmod 8) (type: bigint) - sort order: + - Map-reduce partition columns: (_col0 pmod 8) (type: bigint) - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -632,7 +600,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -647,6 +615,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 pmod 8) (type: bigint) + sort order: + + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -782,7 +782,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -795,7 +795,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: st @@ -811,21 +826,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -898,7 +898,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -911,7 +911,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -927,21 +942,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1014,7 +1014,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1027,7 +1027,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -1043,21 +1058,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1130,7 +1130,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1143,7 +1143,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -1159,21 +1174,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1249,7 +1249,7 @@ STAGE PLANS: 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) @@ -1258,39 +1258,7 @@ STAGE PLANS: Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: sm - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col0 pmod 8) (type: bigint) - sort order: + - Map-reduce partition columns: (_col0 pmod 8) (type: bigint) - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cmodtinyint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1309,7 +1277,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1324,6 +1292,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 pmod 8) (type: bigint) + sort order: + + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out index 736b8f9..420cf28 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out @@ -139,10 +139,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -159,7 +159,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) @@ -173,7 +173,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) @@ -188,7 +188,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: tjoin3 @@ -264,10 +264,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -284,7 +284,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) @@ -298,7 +298,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -309,7 +309,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -326,7 +326,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: tjoin3 diff --git ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out index 8b054b8..991cd5f 100644 --- ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out +++ ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out @@ -110,7 +110,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -234,7 +234,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -358,7 +358,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -469,7 +469,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -580,7 +580,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out index 2703aff..e0030d4 100644 --- ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out +++ ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out @@ -259,11 +259,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny_orc @@ -280,7 +280,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -296,7 +296,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -368,10 +368,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny_orc @@ -394,7 +394,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1021,11 +1021,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_date @@ -1042,7 +1042,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1058,7 +1058,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 1400 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1154,10 +1154,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_date @@ -1180,7 +1180,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1831,11 +1831,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_timestamp @@ -1852,7 +1852,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1868,7 +1868,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 1000 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1964,10 +1964,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_timestamp @@ -1990,7 +1990,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_reduce2.q.out ql/src/test/results/clientpositive/llap/vector_reduce2.q.out index fd10498..0676b69 100644 --- ql/src/test/results/clientpositive/llap/vector_reduce2.q.out +++ ql/src/test/results/clientpositive/llap/vector_reduce2.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -134,7 +134,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_reduce3.q.out ql/src/test/results/clientpositive/llap/vector_reduce3.q.out index 530eb58..2448d70 100644 --- ql/src/test/results/clientpositive/llap/vector_reduce3.q.out +++ ql/src/test/results/clientpositive/llap/vector_reduce3.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -134,7 +134,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_reduce_groupby_decimal.q.out ql/src/test/results/clientpositive/llap/vector_reduce_groupby_decimal.q.out index 9571b5b..49a04e9 100644 --- ql/src/test/results/clientpositive/llap/vector_reduce_groupby_decimal.q.out +++ ql/src/test/results/clientpositive/llap/vector_reduce_groupby_decimal.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_test @@ -64,7 +64,7 @@ STAGE PLANS: value expressions: _col4 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -79,7 +79,7 @@ STAGE PLANS: Statistics: Num rows: 3051 Data size: 720036 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col4 (type: decimal(20,10)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_string_concat.q.out ql/src/test/results/clientpositive/llap/vector_string_concat.q.out index d2672de..a386ecc 100644 --- ql/src/test/results/clientpositive/llap/vector_string_concat.q.out +++ ql/src/test/results/clientpositive/llap/vector_string_concat.q.out @@ -282,11 +282,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -308,7 +308,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -321,7 +321,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_udf1.q.out ql/src/test/results/clientpositive/llap/vector_udf1.q.out index 0dd278e..de3776e 100644 --- ql/src/test/results/clientpositive/llap/vector_udf1.q.out +++ ql/src/test/results/clientpositive/llap/vector_udf1.q.out @@ -53,7 +53,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -121,7 +121,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -189,7 +189,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -257,7 +257,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -325,7 +325,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -393,7 +393,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -461,7 +461,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -529,7 +529,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -597,7 +597,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -665,7 +665,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -733,7 +733,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -801,7 +801,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -869,7 +869,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -937,7 +937,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1005,7 +1005,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1073,7 +1073,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1141,7 +1141,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1209,7 +1209,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1277,7 +1277,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1345,7 +1345,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1413,7 +1413,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1479,7 +1479,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1543,7 +1543,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1607,7 +1607,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1673,7 +1673,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1741,7 +1741,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1807,10 +1807,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1830,7 +1830,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: struct) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1888,10 +1888,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1911,7 +1911,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: varchar(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1969,10 +1969,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -1992,7 +1992,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: varchar(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out index 7d14256..03c3340 100644 --- ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out @@ -138,7 +138,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc diff --git ql/src/test/results/clientpositive/llap/vector_varchar_mapjoin1.q.out ql/src/test/results/clientpositive/llap/vector_varchar_mapjoin1.q.out index 68dd80f..08e5ebb 100644 --- ql/src/test/results/clientpositive/llap/vector_varchar_mapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/vector_varchar_mapjoin1.q.out @@ -137,11 +137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -161,7 +161,7 @@ STAGE PLANS: 1 _col1 (type: varchar(10)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -170,7 +170,21 @@ STAGE PLANS: value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: varchar(10)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: varchar(10)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -190,20 +204,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: varchar(10)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -235,11 +235,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: 1 _col1 (type: varchar(20)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -268,7 +268,21 @@ STAGE PLANS: value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: varchar(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: varchar(20)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -288,20 +302,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: varchar(20)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -335,11 +335,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -359,7 +359,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -368,7 +368,21 @@ STAGE PLANS: value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -388,20 +402,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 300 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out index 092a2ea..18ee310 100644 --- ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out +++ ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out @@ -29,10 +29,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -55,7 +55,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_0.q.out ql/src/test/results/clientpositive/llap/vectorization_0.q.out index 64ac81c..c2e0913 100644 --- ql/src/test/results/clientpositive/llap/vectorization_0.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_0.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -47,7 +47,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: tinyint), _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -60,7 +60,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: tinyint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -119,11 +119,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -143,7 +143,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -155,7 +155,7 @@ STAGE PLANS: key expressions: _col0 (type: bigint) sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -224,11 +224,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -248,7 +248,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -261,7 +261,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -336,11 +336,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -360,7 +360,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -373,7 +373,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -432,11 +432,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -456,7 +456,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -468,7 +468,7 @@ STAGE PLANS: key expressions: _col0 (type: bigint) sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -537,11 +537,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -561,7 +561,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -574,7 +574,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -649,11 +649,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -673,7 +673,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -686,7 +686,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: float), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -745,11 +745,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -769,7 +769,7 @@ STAGE PLANS: value expressions: _col0 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -781,7 +781,7 @@ STAGE PLANS: key expressions: _col0 (type: double) sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -850,11 +850,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -874,7 +874,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -887,7 +887,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1001,10 +1001,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1027,7 +1027,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: bigint), _col4 (type: double), _col5 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1130,10 +1130,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1210,7 +1210,7 @@ STAGE PLANS: name: default.alltypesorc Truncated Path -> Alias: /alltypesorc [alltypesorc] - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Needs Tagging: false Reduce Operator Tree: @@ -30027,11 +30027,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -30109,7 +30109,7 @@ STAGE PLANS: name: default.alltypesorc Truncated Path -> Alias: /alltypesorc [alltypesorc] - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Needs Tagging: false Reduce Operator Tree: @@ -30131,7 +30131,7 @@ STAGE PLANS: tag: -1 value expressions: _col0 (type: bigint) auto parallelism: false - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/vectorization_13.q.out ql/src/test/results/clientpositive/llap/vectorization_13.q.out index 71aa76f..67e26ad 100644 --- ql/src/test/results/clientpositive/llap/vectorization_13.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_13.q.out @@ -73,11 +73,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -103,7 +103,7 @@ STAGE PLANS: value expressions: _col5 (type: tinyint), _col6 (type: double), _col7 (type: struct), _col8 (type: struct), _col9 (type: float), _col10 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,7 +121,7 @@ STAGE PLANS: sort order: +++++++++++++++++++++ Statistics: Num rows: 1365 Data size: 446640 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -328,11 +328,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -358,7 +358,7 @@ STAGE PLANS: value expressions: _col5 (type: tinyint), _col6 (type: double), _col7 (type: struct), _col8 (type: struct), _col9 (type: float), _col10 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -376,7 +376,7 @@ STAGE PLANS: sort order: +++++++++++++++++++++ Statistics: Num rows: 1365 Data size: 446640 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_14.q.out ql/src/test/results/clientpositive/llap/vectorization_14.q.out index 8ba8413..576de56 100644 --- ql/src/test/results/clientpositive/llap/vectorization_14.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_14.q.out @@ -73,11 +73,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -103,7 +103,7 @@ STAGE PLANS: value expressions: _col5 (type: struct), _col6 (type: float), _col7 (type: struct), _col8 (type: bigint), _col9 (type: struct), _col10 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,7 +121,7 @@ STAGE PLANS: sort order: ++++ Statistics: Num rows: 151 Data size: 36700 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: float), _col10 (type: float), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: bigint), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: double), _col19 (type: double), _col20 (type: double), _col21 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_15.q.out ql/src/test/results/clientpositive/llap/vectorization_15.q.out index 04cd902..d9ef0e2 100644 --- ql/src/test/results/clientpositive/llap/vectorization_15.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_15.q.out @@ -69,11 +69,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -99,7 +99,7 @@ STAGE PLANS: value expressions: _col7 (type: struct), _col8 (type: double), _col9 (type: struct), _col10 (type: struct), _col11 (type: struct), _col12 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -117,7 +117,7 @@ STAGE PLANS: sort order: +++++++ Statistics: Num rows: 3072 Data size: 1327460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col7 (type: double), _col8 (type: decimal(13,2)), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: tinyint), _col16 (type: double), _col17 (type: float), _col18 (type: int), _col19 (type: decimal(13,2)), _col20 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_16.q.out ql/src/test/results/clientpositive/llap/vectorization_16.q.out index 622ac88..06c69ee 100644 --- ql/src/test/results/clientpositive/llap/vectorization_16.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_16.q.out @@ -47,10 +47,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -76,7 +76,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: struct), _col5 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_17.q.out ql/src/test/results/clientpositive/llap/vectorization_17.q.out index 94e17b0..9e37fa9 100644 --- ql/src/test/results/clientpositive/llap/vectorization_17.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_17.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -77,7 +77,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: timestamp), _col4 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: decimal(11,4)), _col13 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_8.q.out ql/src/test/results/clientpositive/llap/vectorization_8.q.out index f1bb8c3..d99a9b0 100644 --- ql/src/test/results/clientpositive/llap/vectorization_8.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_8.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -79,7 +79,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -234,10 +234,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -256,7 +256,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out index 2078e81..25cbb38 100644 --- ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out @@ -59,10 +59,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_part @@ -78,7 +78,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_pushdown.q.out ql/src/test/results/clientpositive/llap/vectorization_pushdown.q.out index 4a59aa4..a11a021 100644 --- ql/src/test/results/clientpositive/llap/vectorization_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_pushdown.q.out @@ -12,10 +12,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -39,7 +39,7 @@ STAGE PLANS: value expressions: _col0 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out index 908e8ab..981c3a3 100644 --- ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out @@ -79,10 +79,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -105,7 +105,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: double), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: tinyint), _col8 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -283,10 +283,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -309,7 +309,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: struct), _col2 (type: struct), _col3 (type: double), _col4 (type: struct), _col5 (type: int), _col6 (type: double), _col7 (type: struct), _col8 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,10 +479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -505,7 +505,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: struct), _col4 (type: int), _col5 (type: struct), _col6 (type: bigint), _col7 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -654,10 +654,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -680,7 +680,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: bigint), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -837,10 +837,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -859,7 +859,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1088,10 +1088,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1110,7 +1110,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1288,10 +1288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1311,7 +1311,7 @@ STAGE PLANS: value expressions: _col2 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1546,10 +1546,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1569,7 +1569,7 @@ STAGE PLANS: value expressions: _col0 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1745,11 +1745,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1775,7 +1775,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: bigint), _col3 (type: struct), _col4 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1793,7 +1793,7 @@ STAGE PLANS: sort order: +++++++++++ Statistics: Num rows: 1128 Data size: 197388 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1945,11 +1945,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -1975,7 +1975,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct), _col5 (type: struct), _col6 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1993,7 +1993,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 870 Data size: 109608 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2192,11 +2192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -2222,7 +2222,7 @@ STAGE PLANS: value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: bigint), _col5 (type: tinyint), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct), _col11 (type: double), _col12 (type: struct), _col13 (type: struct), _col14 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2240,7 +2240,7 @@ STAGE PLANS: sort order: +++++++++++++++++++++++++++++++++++++++ Statistics: Num rows: 3072 Data size: 1542740 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2517,11 +2517,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -2547,7 +2547,7 @@ STAGE PLANS: value expressions: _col1 (type: float), _col2 (type: bigint), _col3 (type: struct), _col4 (type: struct), _col5 (type: bigint), _col6 (type: struct), _col7 (type: bigint), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2565,7 +2565,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 3 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: float), _col2 (type: float), _col3 (type: double), _col4 (type: bigint), _col5 (type: decimal(23,3)), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: double), _col13 (type: float), _col14 (type: double), _col15 (type: double), _col17 (type: bigint), _col18 (type: double), _col19 (type: decimal(24,3)), _col20 (type: decimal(25,3)), _col21 (type: double), _col22 (type: decimal(25,3)), _col23 (type: double), _col24 (type: double), _col25 (type: double) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2695,10 +2695,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_count @@ -2716,7 +2716,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2762,10 +2762,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_count @@ -2785,7 +2785,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2905,10 +2905,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesnullorc @@ -2926,7 +2926,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2972,10 +2972,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesnullorc @@ -2995,7 +2995,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3041,10 +3041,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesnullorc @@ -3064,7 +3064,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3110,10 +3110,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesnullorc @@ -3133,7 +3133,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3179,10 +3179,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesnullorc @@ -3202,7 +3202,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3248,10 +3248,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesnullorc @@ -3271,7 +3271,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out index d0efe00..465be50 100644 --- ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_bucketmapjoin1.q.out @@ -109,10 +109,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -128,23 +128,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -166,6 +150,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -202,10 +202,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -221,23 +221,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -259,6 +243,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -295,10 +295,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -314,23 +314,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 52 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2 Data size: 52 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 52 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -352,6 +336,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 52 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_case.q.out ql/src/test/results/clientpositive/llap/vectorized_case.q.out index 6e13369..b6effa4 100644 --- ql/src/test/results/clientpositive/llap/vectorized_case.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_case.q.out @@ -43,7 +43,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -158,7 +158,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc diff --git ql/src/test/results/clientpositive/llap/vectorized_context.q.out ql/src/test/results/clientpositive/llap/vectorized_context.q.out index 1f70a01..00133cc 100644 --- ql/src/test/results/clientpositive/llap/vectorized_context.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_context.q.out @@ -105,10 +105,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -128,7 +128,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: store @@ -148,7 +148,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6682 Data size: 80009 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -158,7 +158,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col2, _col4 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 7350 Data size: 88009 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: string), _col2 (type: double) @@ -176,7 +176,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: household_demographics diff --git ql/src/test/results/clientpositive/llap/vectorized_date_funcs.q.out ql/src/test/results/clientpositive/llap/vectorized_date_funcs.q.out index 40f5b73..975a690 100644 --- ql/src/test/results/clientpositive/llap/vectorized_date_funcs.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_date_funcs.q.out @@ -1060,11 +1060,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: date_udf_flight_orc @@ -1084,7 +1084,7 @@ STAGE PLANS: value expressions: _col0 (type: date), _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1097,7 +1097,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: date), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out index 29f2391..df41409 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out @@ -37,13 +37,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -63,40 +63,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key_int (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -115,7 +82,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -130,7 +97,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key_int is not null (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key_int is not null (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key_int (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -173,13 +173,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -199,40 +199,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key_str (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -251,7 +218,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -266,7 +233,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key_str is not null (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key_str is not null (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key_str (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -309,13 +309,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -335,40 +335,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key_str (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -387,7 +354,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -402,7 +369,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key_str is not null (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key_str is not null (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key_str (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -445,14 +445,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -472,7 +472,43 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + Statistics: Num rows: 1100 Data size: 198000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -505,7 +541,19 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 + Vertex 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=57) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Vertex 6 Map Operator Tree: TableScan alias: c @@ -538,55 +586,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - Statistics: Num rows: 1100 Data size: 198000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=57) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 7 + Vertex 7 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -629,14 +629,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE), Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -656,7 +656,41 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: int) + 1 _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 550 Data size: 99000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -702,41 +736,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: int) - 1 _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 550 Data size: 99000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -748,7 +748,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 6 + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -791,13 +791,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -817,40 +817,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - filterExpr: ((value) IN ('nonexistent1', 'nonexistent2') and key_int is not null) (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((value) IN ('nonexistent1', 'nonexistent2') and key_int is not null) (type: boolean) - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key_int (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=29) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -869,7 +836,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -884,7 +851,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: ((value) IN ('nonexistent1', 'nonexistent2') and key_int is not null) (type: boolean) + Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('nonexistent1', 'nonexistent2') and key_int is not null) (type: boolean) + Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key_int (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=29) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_join46.q.out ql/src/test/results/clientpositive/llap/vectorized_join46.q.out index 1bd4a60..fef6fc1 100644 --- ql/src/test/results/clientpositive/llap/vectorized_join46.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_join46.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,7 +75,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -86,7 +86,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -155,10 +155,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -178,7 +178,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -189,7 +189,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -240,7 +240,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -262,10 +262,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -285,7 +285,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 116 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -296,7 +296,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -321,7 +321,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -363,10 +363,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -383,7 +383,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -400,7 +400,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -438,7 +438,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -458,10 +458,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -481,7 +481,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -492,7 +492,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -514,7 +514,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -541,7 +541,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -565,10 +565,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -585,7 +585,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -597,7 +597,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -619,7 +619,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -651,7 +651,7 @@ NULL NULL None 102 2 Del 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -673,10 +673,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -693,7 +693,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -705,7 +705,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -727,7 +727,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -757,7 +757,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -779,10 +779,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -799,7 +799,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -811,7 +811,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -833,7 +833,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -881,10 +881,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -901,7 +901,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -913,7 +913,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -962,7 +962,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -986,10 +986,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1004,7 +1004,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1021,7 +1021,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1040,7 +1040,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1072,7 +1072,7 @@ NULL NULL None 102 2 Del 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1094,10 +1094,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1112,7 +1112,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1129,7 +1129,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1148,7 +1148,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1175,7 +1175,7 @@ POSTHOOK: Input: default@test2 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1197,10 +1197,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1215,7 +1215,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1232,7 +1232,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1251,7 +1251,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1301,10 +1301,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1321,7 +1321,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1338,7 +1338,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1380,7 +1380,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1404,10 +1404,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1422,22 +1422,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1456,6 +1441,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1463,7 +1463,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1495,7 +1495,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1517,10 +1517,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1535,22 +1535,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1569,6 +1554,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1576,7 +1576,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1606,7 +1606,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1628,10 +1628,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1646,22 +1646,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1680,6 +1665,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1687,7 +1687,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1737,10 +1737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1757,24 +1757,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1793,6 +1776,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out index b40acf9..1b2b554 100644 --- ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out @@ -133,11 +133,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -205,7 +205,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -236,7 +236,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -382,12 +382,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -459,78 +459,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [p1] - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Execution mode: vectorized, llap - LLAP IO: all inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: part_orc - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment - columns.comments - columns.types int:string:string:string:string:int:string:double:string -#### A masked pattern was here #### - name default.part_orc - numFiles 1 - numRows 26 - rawDataSize 16042 - serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 2689 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment - columns.comments - columns.types int:string:string:string:string:int:string:double:string -#### A masked pattern was here #### - name default.part_orc - numFiles 1 - numRows 26 - rawDataSize 16042 - serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 2689 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.part_orc - name: default.part_orc - Truncated Path -> Alias: - /part_orc [p2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -552,7 +481,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -583,7 +512,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -638,6 +567,77 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: all inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: part_orc + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment + columns.comments + columns.types int:string:string:string:string:int:string:double:string +#### A masked pattern was here #### + name default.part_orc + numFiles 1 + numRows 26 + rawDataSize 16042 + serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 2689 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment + columns.comments + columns.types int:string:string:string:string:int:string:double:string +#### A masked pattern was here #### + name default.part_orc + numFiles 1 + numRows 26 + rawDataSize 16042 + serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 2689 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.part_orc + name: default.part_orc + Truncated Path -> Alias: + /part_orc [p2] Stage: Stage-0 Fetch Operator @@ -710,10 +710,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -781,7 +781,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -905,11 +905,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -977,7 +977,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1008,7 +1008,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1160,11 +1160,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -1232,7 +1232,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1263,7 +1263,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1418,11 +1418,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -1490,7 +1490,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1529,7 +1529,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1678,11 +1678,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -1750,7 +1750,77 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 ASC NULLS FIRST + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Filter Operator + isSamplingPred: false + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 p_partkey (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8 + columns.types int:string:string:string:string:int:string:double:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: p1 @@ -1821,76 +1891,6 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [p1] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - PTF Operator - Function definitions: - Input definition - input alias: part_orc - output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string - type: TABLE - Partition table definition - input alias: abc - name: noop - order by: _col1 ASC NULLS FIRST - output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string - partition by: _col2 - raw input shape: - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - Filter Operator - isSamplingPred: false - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 p_partkey (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8 - columns.types int:string:string:string:string:int:string:double:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1965,11 +1965,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -2040,7 +2040,46 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [p1] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 p_partkey (type: int) + 1 _col0 (type: int) + outputColumnNames: _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Position of Big Table: 1 + Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8 + columns.types int:string:string:string:string:int:string:double:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: part_orc @@ -2108,46 +2147,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 p_partkey (type: int) - 1 _col0 (type: int) - outputColumnNames: _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 - Position of Big Table: 1 - Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col12 (type: int), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: int), _col18 (type: string), _col19 (type: double), _col20 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 17646 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8 - columns.types int:string:string:string:string:int:string:double:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -2256,11 +2256,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -2343,7 +2343,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -2374,7 +2374,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -2505,11 +2505,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -2593,7 +2593,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -2625,7 +2625,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -2773,11 +2773,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -2845,7 +2845,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -2876,7 +2876,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3026,12 +3026,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -3099,7 +3099,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3153,7 +3153,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3192,7 +3192,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3350,11 +3350,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -3422,7 +3422,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3453,7 +3453,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3607,12 +3607,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -3680,78 +3680,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: p_partkey (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Execution mode: vectorized, llap - LLAP IO: all inputs - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: part_orc - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment - columns.comments - columns.types int:string:string:string:string:int:string:double:string -#### A masked pattern was here #### - name default.part_orc - numFiles 1 - numRows 26 - rawDataSize 16042 - serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 2689 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment - columns.comments - columns.types int:string:string:string:string:int:string:double:string -#### A masked pattern was here #### - name default.part_orc - numFiles 1 - numRows 26 - rawDataSize 16042 - serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 2689 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.part_orc - name: default.part_orc - Truncated Path -> Alias: - /part_orc [p1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3786,7 +3715,7 @@ STAGE PLANS: tag: 0 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3808,7 +3737,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -3889,6 +3818,77 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: p_partkey (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: all inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: part_orc + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment + columns.comments + columns.types int:string:string:string:string:int:string:double:string +#### A masked pattern was here #### + name default.part_orc + numFiles 1 + numRows 26 + rawDataSize 16042 + serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 2689 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns p_partkey,p_name,p_mfgr,p_brand,p_type,p_size,p_container,p_retailprice,p_comment + columns.comments + columns.types int:string:string:string:string:int:string:double:string +#### A masked pattern was here #### + name default.part_orc + numFiles 1 + numRows 26 + rawDataSize 16042 + serialization.ddl struct part_orc { i32 p_partkey, string p_name, string p_mfgr, string p_brand, string p_type, i32 p_size, string p_container, double p_retailprice, string p_comment} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 2689 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.part_orc + name: default.part_orc + Truncated Path -> Alias: + /part_orc [p1] Stage: Stage-0 Fetch Operator @@ -3971,11 +3971,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -4043,7 +4043,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -4082,7 +4082,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Needs Tagging: false Reduce Operator Tree: @@ -4205,11 +4205,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -4287,7 +4287,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -4324,7 +4324,7 @@ STAGE PLANS: tag: -1 value expressions: _col2 (type: double) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -4510,13 +4510,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -4584,7 +4584,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -4624,7 +4624,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: string) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -4702,7 +4702,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -4743,7 +4743,7 @@ STAGE PLANS: tag: -1 value expressions: sum_window_0 (type: bigint), _col5 (type: int) auto parallelism: true - Reducer 5 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5057,12 +5057,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -5130,7 +5130,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5191,7 +5191,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5230,7 +5230,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5410,13 +5410,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -5484,7 +5484,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5522,7 +5522,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5553,7 +5553,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5584,7 +5584,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 5 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5760,12 +5760,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -5833,7 +5833,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5871,7 +5871,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -5909,7 +5909,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6085,13 +6085,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -6159,7 +6159,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6197,7 +6197,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6244,7 +6244,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6276,7 +6276,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 5 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6454,12 +6454,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -6527,7 +6527,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6581,7 +6581,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6620,7 +6620,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6792,12 +6792,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_orc @@ -6865,7 +6865,7 @@ STAGE PLANS: name: default.part_orc Truncated Path -> Alias: /part_orc [part_orc] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6919,7 +6919,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -6951,7 +6951,7 @@ STAGE PLANS: tag: -1 value expressions: _col5 (type: int) auto parallelism: true - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/vectorized_timestamp_funcs.q.out ql/src/test/results/clientpositive/llap/vectorized_timestamp_funcs.q.out index 0044841..07a1395 100644 --- ql/src/test/results/clientpositive/llap/vectorized_timestamp_funcs.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_timestamp_funcs.q.out @@ -98,10 +98,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_string @@ -117,7 +117,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -243,10 +243,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_string @@ -262,7 +262,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -388,10 +388,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_string @@ -407,7 +407,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -533,10 +533,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_wrong @@ -552,7 +552,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -629,10 +629,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_string @@ -652,7 +652,7 @@ STAGE PLANS: value expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -710,10 +710,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_string @@ -733,7 +733,7 @@ STAGE PLANS: value expressions: _col0 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -803,10 +803,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc_string @@ -826,7 +826,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/windowing.q.out ql/src/test/results/clientpositive/llap/windowing.q.out index 86804f2..c79d5da 100644 --- ql/src/test/results/clientpositive/llap/windowing.q.out +++ ql/src/test/results/clientpositive/llap/windowing.q.out @@ -1826,11 +1826,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1843,7 +1843,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1884,7 +1884,7 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: bigint) Statistics: Num rows: 13 Data size: 3003 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/windowing_gby.q.out ql/src/test/results/clientpositive/llap/windowing_gby.q.out index a31a4b2..8a9ea4a 100644 --- ql/src/test/results/clientpositive/llap/windowing_gby.q.out +++ ql/src/test/results/clientpositive/llap/windowing_gby.q.out @@ -13,15 +13,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap + Vertex 4 llap File Output Operator [FS_19] Select Operator [SEL_17] (rows=2 width=4) Output:["_col0"] @@ -29,15 +29,15 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(UDFToDouble(_col1) / UDFToDouble(_col2)) ASC NULLS FIRST","partition by:":"0"}] Select Operator [SEL_15] (rows=2 width=16) Output:["_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Vertex 3 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_14] PartitionCols:0 Select Operator [SEL_13] (rows=2 width=20) Output:["_col1","_col2"] Group By Operator [GBY_12] (rows=2 width=20) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] + <-Vertex 2 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_11] PartitionCols:_col0 Group By Operator [GBY_10] (rows=2 width=20) Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)","sum(_col1)"],keys:_col2 @@ -45,8 +45,8 @@ Stage-0 Output:["_col2","_col3","_col1"] Merge Join Operator [MERGEJOIN_24] (rows=29 width=12) Conds:RS_6._col0=RS_7._col1(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] + <-Vertex 1 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=18 width=87) Output:["_col0","_col1","_col2"] @@ -54,8 +54,8 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=20 width=88) default@cbo_t3,ws,Tbl:COMPLETE,Col:COMPLETE,Output:["value","c_int","c_boolean"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] + <-Vertex 5 [SORT_PARTITION_EDGE] llap + PARTITION_SORT [RS_7] PartitionCols:_col1 Select Operator [SEL_5] (rows=9174 width=73) Output:["_col0","_col1"] diff --git ql/src/test/results/clientpositive/perf/query1.q.out ql/src/test/results/clientpositive/perf/query1.q.out index a47548f..17906aa 100644 --- ql/src/test/results/clientpositive/perf/query1.q.out +++ ql/src/test/results/clientpositive/perf/query1.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[119][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 18' is a cross product +Warning: Shuffle Join MERGEJOIN[119][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 18' is a cross product PREHOOK: query: explain with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk @@ -48,202 +48,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Reducer 23 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (PARTITION_EDGE), Vertex 20 (PARTITION_EDGE), Vertex 23 (PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_84] - Limit [LIM_83] (rows=100 width=2) - Number of rows:100 - Select Operator [SEL_82] (rows=435393727038609152 width=2) - Output:["_col0"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_81] - Select Operator [SEL_80] (rows=435393727038609152 width=2) - Output:["_col0"] - Filter Operator [FIL_79] (rows=435393727038609152 width=2) - predicate:(_col2 > CASE WHEN (_col8 is null) THEN (null) ELSE (_col7) END) - Merge Join Operator [MERGEJOIN_125] (rows=1306181181115827456 width=2) - Conds:RS_75._col1=RS_76._col2(Left Outer),Output:["_col2","_col6","_col7","_col8"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col2 - Select Operator [SEL_68] (rows=1187437411641012224 width=2) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_67] (rows=1187437411641012224 width=2) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_65] (rows=2374874823282024448 width=2) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col3 - Select Operator [SEL_64] (rows=2374874823282024448 width=2) - Output:["_col3","_col2"] - Merge Join Operator [MERGEJOIN_124] (rows=2374874823282024448 width=2) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col2","_col3"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_33] (rows=31675133 width=77) - Output:["_col1","_col2"] - Group By Operator [GBY_32] (rows=31675133 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0, _col1 - Group By Operator [GBY_30] (rows=63350266 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_29] (rows=63350266 width=77) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_123] (rows=63350266 width=77) - Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_114] (rows=57591150 width=77) - predicate:sr_returned_date_sk is not null - TableScan [TS_20] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_customer_sk","sr_store_sk","sr_fee"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_115] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Group By Operator [GBY_59] (rows=2158977065280000000 width=2) - Output:["_col0"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=4317954130560000000 width=2) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_119] (rows=4317954130560000000 width=2) - Conds:(Inner),(Inner),Output:["_col3"] - <-Map 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Select Operator [SEL_35] (rows=80000000 width=4) - TableScan [TS_34] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:COMPLETE - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Select Operator [SEL_37] (rows=1704 width=4) - TableScan [TS_36] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 23 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_54] - Select Operator [SEL_51] (rows=31675133 width=77) - Output:["_col1"] - Group By Operator [GBY_50] (rows=31675133 width=77) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0, _col1 - Group By Operator [GBY_48] (rows=63350266 width=77) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_118] (rows=63350266 width=77) - Conds:RS_44._col0=RS_45._col0(Inner),Output:["_col1","_col2"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_40] (rows=57591150 width=77) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_116] (rows=57591150 width=77) - predicate:sr_returned_date_sk is not null - TableScan [TS_38] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_customer_sk","sr_store_sk"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_43] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_117] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_41] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_122] (rows=88000001 width=860) - Conds:RS_72._col0=RS_73._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_113] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_17] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_121] (rows=34842647 width=77) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col0","_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_112] (rows=852 width=1910) - predicate:((s_state = 'NM') and s_store_sk is not null) - TableScan [TS_14] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Select Operator [SEL_13] (rows=31675133 width=77) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_12] (rows=31675133 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=63350266 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=63350266 width=77) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_120] (rows=63350266 width=77) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_110] (rows=57591150 width=77) - predicate:(sr_returned_date_sk is not null and sr_store_sk is not null and sr_customer_sk is not null) - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_customer_sk","sr_store_sk","sr_fee"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_111] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query12.q.out ql/src/test/results/clientpositive/perf/query12.q.out index afe2cd4..dfa9e6e 100644 --- ql/src/test/results/clientpositive/perf/query12.q.out +++ ql/src/test/results/clientpositive/perf/query12.q.out @@ -23,70 +23,15 @@ and date_dim.d_date between cast('2001-01-12' as date) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_26] - Limit [LIM_25] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_24] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_23] - Select Operator [SEL_21] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_20] (rows=21780404 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_19] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Group By Operator [GBY_17] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_15] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col23)"],keys:_col38, _col41, _col49, _col47, _col42 - Select Operator [SEL_14] (rows=43560808 width=135) - Output:["_col38","_col41","_col49","_col47","_col42","_col23"] - Filter Operator [FIL_32] (rows=43560808 width=135) - predicate:((_col3 = _col37) and (_col0 = _col62)) - Merge Join Operator [MERGEJOIN_37] (rows=174243235 width=135) - Conds:RS_9._col0=RS_11.d_date_sk(Inner),Output:["_col0","_col3","_col23","_col37","_col38","_col41","_col42","_col47","_col49","_col62"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:d_date_sk - Filter Operator [FIL_35] (rows=8116 width=1119) - predicate:(d_date_sk is not null and d_date BETWEEN 2001-01-12 AND 2001-02-11 00:00:00.0) - TableScan [TS_2] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_36] (rows=158402938 width=135) - Conds:RS_4.ws_item_sk=RS_6.i_item_sk(Inner),Output:["_col0","_col3","_col23","_col37","_col38","_col41","_col42","_col47","_col49"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:ws_item_sk - Filter Operator [FIL_33] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:i_item_sk - Filter Operator [FIL_34] (rows=231000 width=1436) - predicate:(i_item_sk is not null and (i_category) IN ('Jewelry', 'Sports', 'Books')) - TableScan [TS_1] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query13.q.out ql/src/test/results/clientpositive/perf/query13.q.out index 941a3a1..ea15591 100644 --- ql/src/test/results/clientpositive/perf/query13.q.out +++ ql/src/test/results/clientpositive/perf/query13.q.out @@ -109,109 +109,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 12 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_43] - Group By Operator [GBY_41] (rows=1 width=764) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","sum(VALUE._col3)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_40] - Group By Operator [GBY_39] (rows=1 width=764) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col7)","avg(_col9)","avg(_col10)","sum(_col10)"] - Merge Join Operator [MERGEJOIN_74] (rows=2016666 width=1014) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col7","_col9","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_64] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=1833333 width=1014) - Output:["_col0","_col5","_col7","_col8"] - Filter Operator [FIL_33] (rows=1833333 width=1014) - predicate:(((_col17) IN ('KY', 'GA', 'NM') and _col9 BETWEEN 100 AND 200) or ((_col17) IN ('MT', 'OR', 'IN') and _col9 BETWEEN 150 AND 300) or ((_col17) IN ('WI', 'MO', 'WV') and _col9 BETWEEN 50 AND 250)) - Merge Join Operator [MERGEJOIN_73] (rows=11000000 width=1014) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col0","_col5","_col7","_col8","_col9","_col17"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_29] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_69] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_27] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_26] (rows=3549306 width=88) - Output:["_col0","_col3","_col5","_col7","_col8","_col9"] - Filter Operator [FIL_25] (rows=3549306 width=88) - predicate:(((_col12 = 'M') and (_col13 = '4 yr Degree') and _col6 BETWEEN 100 AND 150 and (_col15 = 3)) or ((_col12 = 'D') and (_col13 = 'Primary') and _col6 BETWEEN 50 AND 100 and (_col15 = 1)) or ((_col12 = 'U') and (_col13 = 'Advanced Degree') and _col6 BETWEEN 150 AND 200 and (_col15 = 1))) - Select Operator [SEL_24] (rows=85183356 width=88) - Output:["_col0","_col3","_col5","_col6","_col7","_col8","_col9","_col12","_col13","_col15"] - Merge Join Operator [MERGEJOIN_72] (rows=85183356 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col4","_col6","_col7","_col8","_col9","_col10","_col12","_col14","_col15"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_68] (rows=1861800 width=385) - predicate:(((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and ((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_71] (rows=77439413 width=88) - Conds:RS_18._col3=RS_19._col0(Inner),Output:["_col1","_col2","_col4","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_67] (rows=7200 width=107) - predicate:(((hd_dep_count = 3) or (hd_dep_count = 1)) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_70] (rows=70399465 width=88) - Conds:RS_15._col0=RS_16._col4(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9","_col10"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_65] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_3] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col4 - Select Operator [SEL_8] (rows=63999513 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_66] (rows=63999513 width=88) - predicate:((ss_sales_price BETWEEN 100 AND 150 or ss_sales_price BETWEEN 50 AND 100 or ss_sales_price BETWEEN 150 AND 200) and (ss_net_profit BETWEEN 100 AND 200 or ss_net_profit BETWEEN 150 AND 300 or ss_net_profit BETWEEN 50 AND 250) and ss_store_sk is not null and ss_cdemo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_quantity","ss_sales_price","ss_ext_sales_price","ss_ext_wholesale_cost","ss_net_profit"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query14.q.out ql/src/test/results/clientpositive/perf/query14.q.out index 21cabbc..41ff9fc 100644 --- ql/src/test/results/clientpositive/perf/query14.q.out +++ ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,9 +1,9 @@ -Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 114' is a cross product -Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 115' is a cross product -Warning: Shuffle Join MERGEJOIN[912][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[913][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product -Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 61' is a cross product -Warning: Shuffle Join MERGEJOIN[915][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 62' is a cross product +Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 61' is a cross product +Warning: Shuffle Join MERGEJOIN[915][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 62' is a cross product +Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 114' is a cross product +Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 115' is a cross product +Warning: Shuffle Join MERGEJOIN[912][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[913][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk @@ -213,1356 +213,87 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 100 <- Map 103 (SIMPLE_EDGE), Reducer 99 (SIMPLE_EDGE) -Reducer 101 <- Reducer 100 (SIMPLE_EDGE), Union 94 (CONTAINS) -Reducer 105 <- Map 104 (SIMPLE_EDGE), Map 108 (SIMPLE_EDGE) -Reducer 106 <- Map 109 (SIMPLE_EDGE), Reducer 105 (SIMPLE_EDGE) -Reducer 107 <- Reducer 106 (SIMPLE_EDGE), Union 94 (CONTAINS) -Reducer 111 <- Map 110 (SIMPLE_EDGE), Map 116 (SIMPLE_EDGE), Union 112 (CONTAINS) -Reducer 113 <- Union 112 (CUSTOM_SIMPLE_EDGE) -Reducer 114 <- Reducer 113 (CUSTOM_SIMPLE_EDGE), Reducer 126 (CUSTOM_SIMPLE_EDGE) -Reducer 115 <- Reducer 114 (CUSTOM_SIMPLE_EDGE), Reducer 137 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 118 <- Map 117 (SIMPLE_EDGE), Map 119 (SIMPLE_EDGE), Union 112 (CONTAINS) -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 121 <- Map 120 (SIMPLE_EDGE), Map 122 (SIMPLE_EDGE), Union 112 (CONTAINS) -Reducer 124 <- Map 123 (SIMPLE_EDGE), Map 127 (SIMPLE_EDGE), Union 125 (CONTAINS) -Reducer 126 <- Union 125 (CUSTOM_SIMPLE_EDGE) -Reducer 129 <- Map 128 (SIMPLE_EDGE), Map 130 (SIMPLE_EDGE), Union 125 (CONTAINS) -Reducer 132 <- Map 131 (SIMPLE_EDGE), Map 133 (SIMPLE_EDGE), Union 125 (CONTAINS) -Reducer 135 <- Map 134 (SIMPLE_EDGE), Map 138 (SIMPLE_EDGE) -Reducer 136 <- Map 139 (SIMPLE_EDGE), Reducer 135 (SIMPLE_EDGE), Reducer 142 (SIMPLE_EDGE) -Reducer 137 <- Reducer 136 (SIMPLE_EDGE) -Reducer 141 <- Map 140 (SIMPLE_EDGE), Reducer 148 (SIMPLE_EDGE) -Reducer 142 <- Reducer 141 (SIMPLE_EDGE) -Reducer 144 <- Map 143 (SIMPLE_EDGE), Map 149 (SIMPLE_EDGE) -Reducer 145 <- Map 150 (SIMPLE_EDGE), Reducer 144 (SIMPLE_EDGE) -Reducer 146 <- Reducer 145 (SIMPLE_EDGE), Union 147 (CONTAINS) -Reducer 148 <- Union 147 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 152 <- Map 151 (SIMPLE_EDGE), Map 155 (SIMPLE_EDGE) -Reducer 153 <- Map 156 (SIMPLE_EDGE), Reducer 152 (SIMPLE_EDGE) -Reducer 154 <- Reducer 153 (SIMPLE_EDGE), Union 147 (CONTAINS) -Reducer 158 <- Map 157 (SIMPLE_EDGE), Map 161 (SIMPLE_EDGE) -Reducer 159 <- Map 162 (SIMPLE_EDGE), Reducer 158 (SIMPLE_EDGE) -Reducer 160 <- Reducer 159 (SIMPLE_EDGE), Union 147 (CONTAINS) -Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 19 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 20 <- Union 19 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE), Union 19 (CONTAINS) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE), Union 19 (CONTAINS) -Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) -Reducer 30 <- Map 33 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 31 <- Reducer 30 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Reducer 42 (SIMPLE_EDGE) -Reducer 36 <- Reducer 35 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (SIMPLE_EDGE), Map 43 (SIMPLE_EDGE) -Reducer 39 <- Map 44 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) -Reducer 40 <- Reducer 39 (SIMPLE_EDGE), Union 41 (CONTAINS) -Reducer 42 <- Union 41 (SIMPLE_EDGE) -Reducer 46 <- Map 45 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE) -Reducer 47 <- Map 50 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE) -Reducer 48 <- Reducer 47 (SIMPLE_EDGE), Union 41 (CONTAINS) -Reducer 5 <- Reducer 20 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 52 <- Map 51 (SIMPLE_EDGE), Map 55 (SIMPLE_EDGE) -Reducer 53 <- Map 56 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 54 <- Reducer 53 (SIMPLE_EDGE), Union 41 (CONTAINS) -Reducer 58 <- Map 57 (SIMPLE_EDGE), Map 63 (SIMPLE_EDGE), Union 59 (CONTAINS) -Reducer 6 <- Reducer 31 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 60 <- Union 59 (CUSTOM_SIMPLE_EDGE) -Reducer 61 <- Reducer 60 (CUSTOM_SIMPLE_EDGE), Reducer 73 (CUSTOM_SIMPLE_EDGE) -Reducer 62 <- Reducer 61 (CUSTOM_SIMPLE_EDGE), Reducer 84 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 65 <- Map 64 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE), Union 59 (CONTAINS) -Reducer 68 <- Map 67 (SIMPLE_EDGE), Map 69 (SIMPLE_EDGE), Union 59 (CONTAINS) -Reducer 71 <- Map 70 (SIMPLE_EDGE), Map 74 (SIMPLE_EDGE), Union 72 (CONTAINS) -Reducer 73 <- Union 72 (CUSTOM_SIMPLE_EDGE) -Reducer 76 <- Map 75 (SIMPLE_EDGE), Map 77 (SIMPLE_EDGE), Union 72 (CONTAINS) -Reducer 79 <- Map 78 (SIMPLE_EDGE), Map 80 (SIMPLE_EDGE), Union 72 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 82 <- Map 81 (SIMPLE_EDGE), Map 85 (SIMPLE_EDGE) -Reducer 83 <- Map 86 (SIMPLE_EDGE), Reducer 82 (SIMPLE_EDGE), Reducer 89 (SIMPLE_EDGE) -Reducer 84 <- Reducer 83 (SIMPLE_EDGE) -Reducer 88 <- Map 87 (SIMPLE_EDGE), Reducer 95 (SIMPLE_EDGE) -Reducer 89 <- Reducer 88 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) -Reducer 91 <- Map 90 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE) -Reducer 92 <- Map 97 (SIMPLE_EDGE), Reducer 91 (SIMPLE_EDGE) -Reducer 93 <- Reducer 92 (SIMPLE_EDGE), Union 94 (CONTAINS) -Reducer 95 <- Union 94 (SIMPLE_EDGE) -Reducer 99 <- Map 102 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) +Vertex 100 <- Vertex 103 (SORT_PARTITION_EDGE), Vertex 99 (SORT_PARTITION_EDGE) +Vertex 101 <- Union 94 (CONTAINS), Vertex 100 (SORT_PARTITION_EDGE) +Vertex 105 <- Vertex 104 (SORT_PARTITION_EDGE), Vertex 108 (SORT_PARTITION_EDGE) +Vertex 106 <- Vertex 105 (SORT_PARTITION_EDGE), Vertex 109 (SORT_PARTITION_EDGE) +Vertex 107 <- Union 94 (CONTAINS), Vertex 106 (SORT_PARTITION_EDGE) +Vertex 111 <- Union 112 (CONTAINS), Vertex 110 (SORT_PARTITION_EDGE), Vertex 116 (SORT_PARTITION_EDGE) +Vertex 113 <- Union 112 (PARTITION_EDGE) +Vertex 114 <- Vertex 113 (PARTITION_EDGE), Vertex 126 (PARTITION_EDGE) +Vertex 115 <- Union 7 (CONTAINS), Vertex 114 (PARTITION_EDGE), Vertex 137 (PARTITION_EDGE) +Vertex 118 <- Union 112 (CONTAINS), Vertex 117 (SORT_PARTITION_EDGE), Vertex 119 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 121 <- Union 112 (CONTAINS), Vertex 120 (SORT_PARTITION_EDGE), Vertex 122 (SORT_PARTITION_EDGE) +Vertex 124 <- Union 125 (CONTAINS), Vertex 123 (SORT_PARTITION_EDGE), Vertex 127 (SORT_PARTITION_EDGE) +Vertex 126 <- Union 125 (PARTITION_EDGE) +Vertex 129 <- Union 125 (CONTAINS), Vertex 128 (SORT_PARTITION_EDGE), Vertex 130 (SORT_PARTITION_EDGE) +Vertex 132 <- Union 125 (CONTAINS), Vertex 131 (SORT_PARTITION_EDGE), Vertex 133 (SORT_PARTITION_EDGE) +Vertex 135 <- Vertex 134 (SORT_PARTITION_EDGE), Vertex 138 (SORT_PARTITION_EDGE) +Vertex 136 <- Vertex 135 (SORT_PARTITION_EDGE), Vertex 139 (SORT_PARTITION_EDGE), Vertex 142 (SORT_PARTITION_EDGE) +Vertex 137 <- Vertex 136 (SORT_PARTITION_EDGE) +Vertex 141 <- Vertex 140 (SORT_PARTITION_EDGE), Vertex 148 (SORT_PARTITION_EDGE) +Vertex 142 <- Vertex 141 (SORT_PARTITION_EDGE) +Vertex 144 <- Vertex 143 (SORT_PARTITION_EDGE), Vertex 149 (SORT_PARTITION_EDGE) +Vertex 145 <- Vertex 144 (SORT_PARTITION_EDGE), Vertex 150 (SORT_PARTITION_EDGE) +Vertex 146 <- Union 147 (CONTAINS), Vertex 145 (SORT_PARTITION_EDGE) +Vertex 148 <- Union 147 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 3 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 152 <- Vertex 151 (SORT_PARTITION_EDGE), Vertex 155 (SORT_PARTITION_EDGE) +Vertex 153 <- Vertex 152 (SORT_PARTITION_EDGE), Vertex 156 (SORT_PARTITION_EDGE) +Vertex 154 <- Union 147 (CONTAINS), Vertex 153 (SORT_PARTITION_EDGE) +Vertex 158 <- Vertex 157 (SORT_PARTITION_EDGE), Vertex 161 (SORT_PARTITION_EDGE) +Vertex 159 <- Vertex 158 (SORT_PARTITION_EDGE), Vertex 162 (SORT_PARTITION_EDGE) +Vertex 160 <- Union 147 (CONTAINS), Vertex 159 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 19 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 19 (PARTITION_EDGE) +Vertex 23 <- Union 19 (CONTAINS), Vertex 22 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 26 <- Union 19 (CONTAINS), Vertex 25 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 42 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE), Vertex 43 (SORT_PARTITION_EDGE) +Vertex 39 <- Vertex 38 (SORT_PARTITION_EDGE), Vertex 44 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (PARTITION_EDGE) +Vertex 40 <- Union 41 (CONTAINS), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 42 <- Union 41 (SORT_PARTITION_EDGE) +Vertex 46 <- Vertex 45 (SORT_PARTITION_EDGE), Vertex 49 (SORT_PARTITION_EDGE) +Vertex 47 <- Vertex 46 (SORT_PARTITION_EDGE), Vertex 50 (SORT_PARTITION_EDGE) +Vertex 48 <- Union 41 (CONTAINS), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 20 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE), Vertex 56 (SORT_PARTITION_EDGE) +Vertex 54 <- Union 41 (CONTAINS), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 58 <- Union 59 (CONTAINS), Vertex 57 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 7 (CONTAINS), Vertex 31 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 60 <- Union 59 (PARTITION_EDGE) +Vertex 61 <- Vertex 60 (PARTITION_EDGE), Vertex 73 (PARTITION_EDGE) +Vertex 62 <- Union 7 (CONTAINS), Vertex 61 (PARTITION_EDGE), Vertex 84 (PARTITION_EDGE) +Vertex 65 <- Union 59 (CONTAINS), Vertex 64 (SORT_PARTITION_EDGE), Vertex 66 (SORT_PARTITION_EDGE) +Vertex 68 <- Union 59 (CONTAINS), Vertex 67 (SORT_PARTITION_EDGE), Vertex 69 (SORT_PARTITION_EDGE) +Vertex 71 <- Union 72 (CONTAINS), Vertex 70 (SORT_PARTITION_EDGE), Vertex 74 (SORT_PARTITION_EDGE) +Vertex 73 <- Union 72 (PARTITION_EDGE) +Vertex 76 <- Union 72 (CONTAINS), Vertex 75 (SORT_PARTITION_EDGE), Vertex 77 (SORT_PARTITION_EDGE) +Vertex 79 <- Union 72 (CONTAINS), Vertex 78 (SORT_PARTITION_EDGE), Vertex 80 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 82 <- Vertex 81 (SORT_PARTITION_EDGE), Vertex 85 (SORT_PARTITION_EDGE) +Vertex 83 <- Vertex 82 (SORT_PARTITION_EDGE), Vertex 86 (SORT_PARTITION_EDGE), Vertex 89 (SORT_PARTITION_EDGE) +Vertex 84 <- Vertex 83 (SORT_PARTITION_EDGE) +Vertex 88 <- Vertex 87 (SORT_PARTITION_EDGE), Vertex 95 (SORT_PARTITION_EDGE) +Vertex 89 <- Vertex 88 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +Vertex 91 <- Vertex 90 (SORT_PARTITION_EDGE), Vertex 96 (SORT_PARTITION_EDGE) +Vertex 92 <- Vertex 91 (SORT_PARTITION_EDGE), Vertex 97 (SORT_PARTITION_EDGE) +Vertex 93 <- Union 94 (CONTAINS), Vertex 92 (SORT_PARTITION_EDGE) +Vertex 95 <- Union 94 (SORT_PARTITION_EDGE) +Vertex 99 <- Vertex 102 (SORT_PARTITION_EDGE), Vertex 98 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 9 - File Output Operator [FS_596] - Limit [LIM_595] (rows=100 width=406) - Number of rows:100 - Select Operator [SEL_594] (rows=1016388080 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_593] - Select Operator [SEL_592] (rows=1016388080 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_591] (rows=1016388080 width=406) - Output:["_col0","_col1","_col2","_col3","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 115 [CONTAINS] - Reduce Output Operator [RS_590] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_589] (rows=2032776160 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_586] (rows=58081078 width=433) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_585] (rows=58081078 width=433) - predicate:(_col5 > _col1) - Merge Join Operator [MERGEJOIN_917] (rows=174243235 width=433) - Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 114 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_582] - Merge Join Operator [MERGEJOIN_916] (rows=1 width=297) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 113 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_579] - Select Operator [SEL_436] (rows=1 width=8) - Filter Operator [FIL_435] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_433] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_428] (rows=1 width=8) - Group By Operator [GBY_427] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Union 112 [CUSTOM_SIMPLE_EDGE] - <-Reducer 111 [CONTAINS] - Reduce Output Operator [RS_426] - Group By Operator [GBY_425] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_424] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_401] (rows=633595212 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_897] (rows=633595212 width=88) - Conds:RS_398._col0=RS_399._col0(Inner),Output:["_col1"] - <-Map 110 [SIMPLE_EDGE] - SHUFFLE [RS_398] - PartitionCols:_col0 - Select Operator [SEL_394] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_842] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_392] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] - <-Map 116 [SIMPLE_EDGE] - SHUFFLE [RS_399] - PartitionCols:_col0 - Select Operator [SEL_397] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_843] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_395] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 118 [CONTAINS] - Reduce Output Operator [RS_426] - Group By Operator [GBY_425] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_424] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_411] (rows=316788826 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_898] (rows=316788826 width=135) - Conds:RS_408._col0=RS_409._col0(Inner),Output:["_col1"] - <-Map 117 [SIMPLE_EDGE] - SHUFFLE [RS_408] - PartitionCols:_col0 - Select Operator [SEL_404] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_844] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_402] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] - <-Map 119 [SIMPLE_EDGE] - SHUFFLE [RS_409] - PartitionCols:_col0 - Select Operator [SEL_407] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_845] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_405] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 121 [CONTAINS] - Reduce Output Operator [RS_426] - Group By Operator [GBY_425] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_424] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_423] (rows=158402938 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_899] (rows=158402938 width=135) - Conds:RS_420._col0=RS_421._col0(Inner),Output:["_col1"] - <-Map 120 [SIMPLE_EDGE] - SHUFFLE [RS_420] - PartitionCols:_col0 - Select Operator [SEL_416] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_846] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_414] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] - <-Map 122 [SIMPLE_EDGE] - SHUFFLE [RS_421] - PartitionCols:_col0 - Select Operator [SEL_419] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_847] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_417] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 126 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_580] - Group By Operator [GBY_473] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Union 125 [CUSTOM_SIMPLE_EDGE] - <-Reducer 124 [CONTAINS] - Reduce Output Operator [RS_472] - Group By Operator [GBY_471] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_469] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_446] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_900] (rows=633595212 width=88) - Conds:RS_443._col0=RS_444._col0(Inner),Output:["_col1","_col2"] - <-Map 123 [SIMPLE_EDGE] - SHUFFLE [RS_443] - PartitionCols:_col0 - Select Operator [SEL_439] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_848] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_437] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] - <-Map 127 [SIMPLE_EDGE] - SHUFFLE [RS_444] - PartitionCols:_col0 - Select Operator [SEL_442] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_849] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_440] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 129 [CONTAINS] - Reduce Output Operator [RS_472] - Group By Operator [GBY_471] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_469] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_456] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_901] (rows=316788826 width=135) - Conds:RS_453._col0=RS_454._col0(Inner),Output:["_col1","_col2"] - <-Map 128 [SIMPLE_EDGE] - SHUFFLE [RS_453] - PartitionCols:_col0 - Select Operator [SEL_449] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_850] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_447] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] - <-Map 130 [SIMPLE_EDGE] - SHUFFLE [RS_454] - PartitionCols:_col0 - Select Operator [SEL_452] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_851] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_450] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 132 [CONTAINS] - Reduce Output Operator [RS_472] - Group By Operator [GBY_471] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_469] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_468] (rows=158402938 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_902] (rows=158402938 width=135) - Conds:RS_465._col0=RS_466._col0(Inner),Output:["_col1","_col2"] - <-Map 131 [SIMPLE_EDGE] - SHUFFLE [RS_465] - PartitionCols:_col0 - Select Operator [SEL_461] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_852] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_459] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] - <-Map 133 [SIMPLE_EDGE] - SHUFFLE [RS_466] - PartitionCols:_col0 - Select Operator [SEL_464] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_853] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_462] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 137 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_583] - Group By Operator [GBY_577] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 136 [SIMPLE_EDGE] - SHUFFLE [RS_576] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_575] (rows=348486471 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_573] (rows=348486471 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_911] (rows=348486471 width=135) - Conds:RS_569._col1=RS_570._col0(Inner),RS_569._col1=RS_571._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"] - <-Map 139 [SIMPLE_EDGE] - SHUFFLE [RS_570] - PartitionCols:_col0 - Select Operator [SEL_483] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_856] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_481] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 135 [SIMPLE_EDGE] - SHUFFLE [RS_569] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_903] (rows=158402938 width=135) - Conds:RS_566._col0=RS_567._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 134 [SIMPLE_EDGE] - SHUFFLE [RS_566] - PartitionCols:_col0 - Select Operator [SEL_477] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_854] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_475] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_quantity","ws_list_price"] - <-Map 138 [SIMPLE_EDGE] - SHUFFLE [RS_567] - PartitionCols:_col0 - Select Operator [SEL_480] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_855] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_478] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 142 [SIMPLE_EDGE] - SHUFFLE [RS_571] - PartitionCols:_col0 - Group By Operator [GBY_564] (rows=254100 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Reducer 141 [SIMPLE_EDGE] - SHUFFLE [RS_563] - PartitionCols:_col0 - Group By Operator [GBY_562] (rows=508200 width=1436) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_910] (rows=508200 width=1436) - Conds:RS_558._col1, _col2, _col3=RS_559._col0, _col1, _col2(Inner),Output:["_col0"] - <-Map 140 [SIMPLE_EDGE] - SHUFFLE [RS_558] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_486] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_857] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) - TableScan [TS_484] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 148 [SIMPLE_EDGE] - SHUFFLE [RS_559] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_557] (rows=1 width=108) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_556] (rows=1 width=108) - predicate:(_col3 = 3) - Group By Operator [GBY_555] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 147 [SIMPLE_EDGE] - <-Reducer 146 [CONTAINS] - Reduce Output Operator [RS_554] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_553] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_506] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 145 [SIMPLE_EDGE] - SHUFFLE [RS_505] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_504] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_502] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_905] (rows=696954748 width=88) - Conds:RS_499._col1=RS_500._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 150 [SIMPLE_EDGE] - SHUFFLE [RS_500] - PartitionCols:_col0 - Select Operator [SEL_495] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_860] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_493] (rows=462000 width=1436) - default@item,iss,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 144 [SIMPLE_EDGE] - SHUFFLE [RS_499] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_904] (rows=633595212 width=88) - Conds:RS_496._col0=RS_497._col0(Inner),Output:["_col1"] - <-Map 143 [SIMPLE_EDGE] - SHUFFLE [RS_496] - PartitionCols:_col0 - Select Operator [SEL_489] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_858] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_487] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 149 [SIMPLE_EDGE] - SHUFFLE [RS_497] - PartitionCols:_col0 - Select Operator [SEL_492] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_859] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_490] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 154 [CONTAINS] - Reduce Output Operator [RS_554] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_553] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_527] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 153 [SIMPLE_EDGE] - SHUFFLE [RS_526] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_525] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_523] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_907] (rows=348467716 width=135) - Conds:RS_520._col1=RS_521._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 156 [SIMPLE_EDGE] - SHUFFLE [RS_521] - PartitionCols:_col0 - Select Operator [SEL_516] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_863] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_514] (rows=462000 width=1436) - default@item,ics,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 152 [SIMPLE_EDGE] - SHUFFLE [RS_520] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_906] (rows=316788826 width=135) - Conds:RS_517._col0=RS_518._col0(Inner),Output:["_col1"] - <-Map 151 [SIMPLE_EDGE] - SHUFFLE [RS_517] - PartitionCols:_col0 - Select Operator [SEL_510] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_861] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_508] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Map 155 [SIMPLE_EDGE] - SHUFFLE [RS_518] - PartitionCols:_col0 - Select Operator [SEL_513] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_862] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_511] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 160 [CONTAINS] - Reduce Output Operator [RS_554] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_553] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_549] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 159 [SIMPLE_EDGE] - SHUFFLE [RS_548] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_547] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_545] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_909] (rows=174243235 width=135) - Conds:RS_542._col1=RS_543._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 162 [SIMPLE_EDGE] - SHUFFLE [RS_543] - PartitionCols:_col0 - Select Operator [SEL_538] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_866] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_536] (rows=462000 width=1436) - default@item,iws,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 158 [SIMPLE_EDGE] - SHUFFLE [RS_542] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_908] (rows=158402938 width=135) - Conds:RS_539._col0=RS_540._col0(Inner),Output:["_col1"] - <-Map 157 [SIMPLE_EDGE] - SHUFFLE [RS_539] - PartitionCols:_col0 - Select Operator [SEL_532] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_864] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_530] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] - <-Map 161 [SIMPLE_EDGE] - SHUFFLE [RS_540] - PartitionCols:_col0 - Select Operator [SEL_535] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_865] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_533] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_590] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_589] (rows=2032776160 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_194] (rows=232318249 width=386) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_193] (rows=232318249 width=386) - predicate:(_col5 > _col1) - Merge Join Operator [MERGEJOIN_913] (rows=696954748 width=386) - Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 31 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_191] - Group By Operator [GBY_185] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_184] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_183] (rows=1393909496 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_181] (rows=1393909496 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_881] (rows=1393909496 width=88) - Conds:RS_177._col1=RS_178._col0(Inner),RS_177._col1=RS_179._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_806] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_89] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_177] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_873] (rows=633595212 width=88) - Conds:RS_174._col0=RS_175._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_174] - PartitionCols:_col0 - Select Operator [SEL_85] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_804] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_83] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_quantity","ss_list_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_805] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_86] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_179] - PartitionCols:_col0 - Group By Operator [GBY_172] (rows=254100 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col0 - Group By Operator [GBY_170] (rows=508200 width=1436) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_880] (rows=508200 width=1436) - Conds:RS_166._col1, _col2, _col3=RS_167._col0, _col1, _col2(Inner),Output:["_col0"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_166] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_94] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_807] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) - TableScan [TS_92] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 42 [SIMPLE_EDGE] - SHUFFLE [RS_167] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_165] (rows=1 width=108) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_164] (rows=1 width=108) - predicate:(_col3 = 3) - Group By Operator [GBY_163] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 41 [SIMPLE_EDGE] - <-Reducer 40 [CONTAINS] - Reduce Output Operator [RS_162] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_161] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_114] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 39 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_112] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_110] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_875] (rows=696954748 width=88) - Conds:RS_107._col1=RS_108._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 44 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_103] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_810] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_101] (rows=462000 width=1436) - default@item,iss,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_874] (rows=633595212 width=88) - Conds:RS_104._col0=RS_105._col0(Inner),Output:["_col1"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Select Operator [SEL_97] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_808] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_95] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 43 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_809] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_98] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 48 [CONTAINS] - Reduce Output Operator [RS_162] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_161] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_135] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 47 [SIMPLE_EDGE] - SHUFFLE [RS_134] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_133] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_131] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_877] (rows=348467716 width=135) - Conds:RS_128._col1=RS_129._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 50 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col0 - Select Operator [SEL_124] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_813] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_122] (rows=462000 width=1436) - default@item,ics,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 46 [SIMPLE_EDGE] - SHUFFLE [RS_128] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_876] (rows=316788826 width=135) - Conds:RS_125._col0=RS_126._col0(Inner),Output:["_col1"] - <-Map 45 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0 - Select Operator [SEL_118] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_811] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_116] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Map 49 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Select Operator [SEL_121] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_812] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_119] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 54 [CONTAINS] - Reduce Output Operator [RS_162] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_161] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_157] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_156] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_155] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_153] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_879] (rows=174243235 width=135) - Conds:RS_150._col1=RS_151._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_151] - PartitionCols:_col0 - Select Operator [SEL_146] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_816] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_144] (rows=462000 width=1436) - default@item,iws,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_150] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_878] (rows=158402938 width=135) - Conds:RS_147._col0=RS_148._col0(Inner),Output:["_col1"] - <-Map 51 [SIMPLE_EDGE] - SHUFFLE [RS_147] - PartitionCols:_col0 - Select Operator [SEL_140] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_814] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_138] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col0 - Select Operator [SEL_143] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_815] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_141] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_190] - Merge Join Operator [MERGEJOIN_912] (rows=1 width=297) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_188] - Group By Operator [GBY_81] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Union 19 [CUSTOM_SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_80] - Group By Operator [GBY_79] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_77] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_54] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_870] (rows=633595212 width=88) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_798] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_45] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_799] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_48] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 23 [CONTAINS] - Reduce Output Operator [RS_80] - Group By Operator [GBY_79] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_77] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_64] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_871] (rows=316788826 width=135) - Conds:RS_61._col0=RS_62._col0(Inner),Output:["_col1","_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_800] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_55] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_60] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_801] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_58] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_80] - Group By Operator [GBY_79] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_77] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_76] (rows=158402938 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_872] (rows=158402938 width=135) - Conds:RS_73._col0=RS_74._col0(Inner),Output:["_col1","_col2"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_802] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_67] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_803] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_70] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_187] - Select Operator [SEL_44] (rows=1 width=8) - Filter Operator [FIL_43] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_41] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_36] (rows=1 width=8) - Group By Operator [GBY_35] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Union 3 [CUSTOM_SIMPLE_EDGE] - <-Reducer 12 [CONTAINS] - Reduce Output Operator [RS_34] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_32] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_19] (rows=316788826 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_868] (rows=316788826 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col1"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_794] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_10] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_795] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_34] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_32] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_31] (rows=158402938 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_869] (rows=158402938 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_796] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_22] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_797] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_34] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_32] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_867] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_792] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_793] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 62 [CONTAINS] - Reduce Output Operator [RS_590] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_589] (rows=2032776160 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_389] (rows=116155905 width=433) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_388] (rows=116155905 width=433) - predicate:(_col5 > _col1) - Merge Join Operator [MERGEJOIN_915] (rows=348467716 width=433) - Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 61 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_385] - Merge Join Operator [MERGEJOIN_914] (rows=1 width=297) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 60 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_382] - Select Operator [SEL_239] (rows=1 width=8) - Filter Operator [FIL_238] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_236] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_231] (rows=1 width=8) - Group By Operator [GBY_230] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Union 59 [CUSTOM_SIMPLE_EDGE] - <-Reducer 58 [CONTAINS] - Reduce Output Operator [RS_229] - Group By Operator [GBY_228] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_227] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_204] (rows=633595212 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_882] (rows=633595212 width=88) - Conds:RS_201._col0=RS_202._col0(Inner),Output:["_col1"] - <-Map 57 [SIMPLE_EDGE] - SHUFFLE [RS_201] - PartitionCols:_col0 - Select Operator [SEL_197] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_817] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_195] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_202] - PartitionCols:_col0 - Select Operator [SEL_200] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_818] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_198] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [CONTAINS] - Reduce Output Operator [RS_229] - Group By Operator [GBY_228] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_227] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_214] (rows=316788826 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_883] (rows=316788826 width=135) - Conds:RS_211._col0=RS_212._col0(Inner),Output:["_col1"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_211] - PartitionCols:_col0 - Select Operator [SEL_207] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_819] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_205] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] - <-Map 66 [SIMPLE_EDGE] - SHUFFLE [RS_212] - PartitionCols:_col0 - Select Operator [SEL_210] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_820] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_208] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 68 [CONTAINS] - Reduce Output Operator [RS_229] - Group By Operator [GBY_228] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_227] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_226] (rows=158402938 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_884] (rows=158402938 width=135) - Conds:RS_223._col0=RS_224._col0(Inner),Output:["_col1"] - <-Map 67 [SIMPLE_EDGE] - SHUFFLE [RS_223] - PartitionCols:_col0 - Select Operator [SEL_219] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_821] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_217] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] - <-Map 69 [SIMPLE_EDGE] - SHUFFLE [RS_224] - PartitionCols:_col0 - Select Operator [SEL_222] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_822] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_220] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 73 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_383] - Group By Operator [GBY_276] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Union 72 [CUSTOM_SIMPLE_EDGE] - <-Reducer 71 [CONTAINS] - Reduce Output Operator [RS_275] - Group By Operator [GBY_274] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_272] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_249] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_885] (rows=633595212 width=88) - Conds:RS_246._col0=RS_247._col0(Inner),Output:["_col1","_col2"] - <-Map 70 [SIMPLE_EDGE] - SHUFFLE [RS_246] - PartitionCols:_col0 - Select Operator [SEL_242] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_823] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_240] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] - <-Map 74 [SIMPLE_EDGE] - SHUFFLE [RS_247] - PartitionCols:_col0 - Select Operator [SEL_245] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_824] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_243] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 76 [CONTAINS] - Reduce Output Operator [RS_275] - Group By Operator [GBY_274] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_272] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_259] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_886] (rows=316788826 width=135) - Conds:RS_256._col0=RS_257._col0(Inner),Output:["_col1","_col2"] - <-Map 75 [SIMPLE_EDGE] - SHUFFLE [RS_256] - PartitionCols:_col0 - Select Operator [SEL_252] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_825] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_250] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] - <-Map 77 [SIMPLE_EDGE] - SHUFFLE [RS_257] - PartitionCols:_col0 - Select Operator [SEL_255] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_826] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_253] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 79 [CONTAINS] - Reduce Output Operator [RS_275] - Group By Operator [GBY_274] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_272] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_271] (rows=158402938 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_887] (rows=158402938 width=135) - Conds:RS_268._col0=RS_269._col0(Inner),Output:["_col1","_col2"] - <-Map 78 [SIMPLE_EDGE] - SHUFFLE [RS_268] - PartitionCols:_col0 - Select Operator [SEL_264] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_827] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_262] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] - <-Map 80 [SIMPLE_EDGE] - SHUFFLE [RS_269] - PartitionCols:_col0 - Select Operator [SEL_267] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_828] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_265] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 84 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_386] - Group By Operator [GBY_380] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 83 [SIMPLE_EDGE] - SHUFFLE [RS_379] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_378] (rows=696935432 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_376] (rows=696935432 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_896] (rows=696935432 width=135) - Conds:RS_372._col1=RS_373._col0(Inner),RS_372._col1=RS_374._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"] - <-Map 86 [SIMPLE_EDGE] - SHUFFLE [RS_373] - PartitionCols:_col0 - Select Operator [SEL_286] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_831] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_284] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 82 [SIMPLE_EDGE] - SHUFFLE [RS_372] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_888] (rows=316788826 width=135) - Conds:RS_369._col0=RS_370._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 81 [SIMPLE_EDGE] - SHUFFLE [RS_369] - PartitionCols:_col0 - Select Operator [SEL_280] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_829] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_278] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_quantity","cs_list_price"] - <-Map 85 [SIMPLE_EDGE] - SHUFFLE [RS_370] - PartitionCols:_col0 - Select Operator [SEL_283] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_830] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_281] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 89 [SIMPLE_EDGE] - SHUFFLE [RS_374] - PartitionCols:_col0 - Group By Operator [GBY_367] (rows=254100 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Reducer 88 [SIMPLE_EDGE] - SHUFFLE [RS_366] - PartitionCols:_col0 - Group By Operator [GBY_365] (rows=508200 width=1436) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_895] (rows=508200 width=1436) - Conds:RS_361._col1, _col2, _col3=RS_362._col0, _col1, _col2(Inner),Output:["_col0"] - <-Map 87 [SIMPLE_EDGE] - SHUFFLE [RS_361] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_289] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_832] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) - TableScan [TS_287] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 95 [SIMPLE_EDGE] - SHUFFLE [RS_362] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_360] (rows=1 width=108) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_359] (rows=1 width=108) - predicate:(_col3 = 3) - Group By Operator [GBY_358] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 94 [SIMPLE_EDGE] - <-Reducer 101 [CONTAINS] - Reduce Output Operator [RS_357] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_356] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_330] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 100 [SIMPLE_EDGE] - SHUFFLE [RS_329] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_328] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_326] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_892] (rows=348467716 width=135) - Conds:RS_323._col1=RS_324._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 103 [SIMPLE_EDGE] - SHUFFLE [RS_324] - PartitionCols:_col0 - Select Operator [SEL_319] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_838] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_317] (rows=462000 width=1436) - default@item,ics,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 99 [SIMPLE_EDGE] - SHUFFLE [RS_323] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_891] (rows=316788826 width=135) - Conds:RS_320._col0=RS_321._col0(Inner),Output:["_col1"] - <-Map 102 [SIMPLE_EDGE] - SHUFFLE [RS_321] - PartitionCols:_col0 - Select Operator [SEL_316] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_837] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_314] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 98 [SIMPLE_EDGE] - SHUFFLE [RS_320] - PartitionCols:_col0 - Select Operator [SEL_313] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_836] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_311] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Reducer 107 [CONTAINS] - Reduce Output Operator [RS_357] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_356] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_352] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 106 [SIMPLE_EDGE] - SHUFFLE [RS_351] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_350] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_348] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_894] (rows=174243235 width=135) - Conds:RS_345._col1=RS_346._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 109 [SIMPLE_EDGE] - SHUFFLE [RS_346] - PartitionCols:_col0 - Select Operator [SEL_341] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_841] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_339] (rows=462000 width=1436) - default@item,iws,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 105 [SIMPLE_EDGE] - SHUFFLE [RS_345] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_893] (rows=158402938 width=135) - Conds:RS_342._col0=RS_343._col0(Inner),Output:["_col1"] - <-Map 104 [SIMPLE_EDGE] - SHUFFLE [RS_342] - PartitionCols:_col0 - Select Operator [SEL_335] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_839] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_333] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] - <-Map 108 [SIMPLE_EDGE] - SHUFFLE [RS_343] - PartitionCols:_col0 - Select Operator [SEL_338] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_840] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_336] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 93 [CONTAINS] - Reduce Output Operator [RS_357] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_356] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_309] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 92 [SIMPLE_EDGE] - SHUFFLE [RS_308] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_307] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_305] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_890] (rows=696954748 width=88) - Conds:RS_302._col1=RS_303._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 97 [SIMPLE_EDGE] - SHUFFLE [RS_303] - PartitionCols:_col0 - Select Operator [SEL_298] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_835] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_296] (rows=462000 width=1436) - default@item,iss,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 91 [SIMPLE_EDGE] - SHUFFLE [RS_302] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_889] (rows=633595212 width=88) - Conds:RS_299._col0=RS_300._col0(Inner),Output:["_col1"] - <-Map 90 [SIMPLE_EDGE] - SHUFFLE [RS_299] - PartitionCols:_col0 - Select Operator [SEL_292] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_833] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_290] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 96 [SIMPLE_EDGE] - SHUFFLE [RS_300] - PartitionCols:_col0 - Select Operator [SEL_295] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_834] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_293] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 9 diff --git ql/src/test/results/clientpositive/perf/query15.q.out ql/src/test/results/clientpositive/perf/query15.q.out index 4f4dcc5..53e9ce7 100644 --- ql/src/test/results/clientpositive/perf/query15.q.out +++ ql/src/test/results/clientpositive/perf/query15.q.out @@ -5,83 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_31] - Limit [LIM_30] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_29] (rows=174233858 width=135) - Output:["_col0","_col1"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Group By Operator [GBY_26] (rows=174233858 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=348467716 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_23] (rows=348467716 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_47] (rows=348467716 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_44] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=316788826 width=135) - Output:["_col0","_col2","_col7"] - Filter Operator [FIL_15] (rows=316788826 width=135) - predicate:((substr(_col4, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792') or (_col3) IN ('CA', 'WA', 'GA') or (_col7 > 500)) - Merge Join Operator [MERGEJOIN_46] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col1(Inner),Output:["_col3","_col4","_col5","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_6] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_45] (rows=88000001 width=860) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_zip"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query16.q.out ql/src/test/results/clientpositive/perf/query16.q.out index 1be5fba..0bf40e8 100644 --- ql/src/test/results/clientpositive/perf/query16.q.out +++ ql/src/test/results/clientpositive/perf/query16.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join MERGEJOIN[141][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 13' is a cross product -Warning: Shuffle Join MERGEJOIN[142][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 25' is a cross product +Warning: Shuffle Join MERGEJOIN[141][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Vertex 13' is a cross product +Warning: Shuffle Join MERGEJOIN[142][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Vertex 25' is a cross product PREHOOK: query: explain select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` @@ -61,216 +61,25 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE), Map 17 (CUSTOM_SIMPLE_EDGE), Map 18 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (CUSTOM_SIMPLE_EDGE), Map 27 (CUSTOM_SIMPLE_EDGE), Map 28 (CUSTOM_SIMPLE_EDGE), Map 29 (CUSTOM_SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 23 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (PARTITION_EDGE), Vertex 17 (PARTITION_EDGE), Vertex 18 (PARTITION_EDGE), Vertex 19 (PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE), Vertex 28 (PARTITION_EDGE), Vertex 29 (PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_94] - Limit [LIM_93] (rows=1 width=344) - Number of rows:100 - Select Operator [SEL_92] (rows=1 width=344) - Output:["_col0","_col1","_col2"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_91] - Group By Operator [GBY_89] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_88] - Group By Operator [GBY_87] (rows=1534538622412395776 width=1) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col4)","sum(_col5)","sum(_col6)"],keys:_col4 - Select Operator [SEL_86] (rows=1534538622412395776 width=1) - Output:["_col4","_col5","_col6"] - Filter Operator [FIL_85] (rows=1534538622412395776 width=1) - predicate:_col16 is null - Select Operator [SEL_84] (rows=3069077244824791552 width=1) - Output:["_col4","_col5","_col6","_col16"] - Merge Join Operator [MERGEJOIN_149] (rows=3069077244824791552 width=1) - Conds:RS_81._col3, _col4=RS_82._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1 - Group By Operator [GBY_67] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],keys:_col2, _col3 - Select Operator [SEL_64] (rows=5072854730221289472 width=1) - Output:["_col2","_col3"] - Filter Operator [FIL_63] (rows=5072854730221289472 width=1) - predicate:(_col2 <> _col0) - Merge Join Operator [MERGEJOIN_147] (rows=5072854730221289472 width=1) - Conds:RS_60._col1=RS_61._col1(Inner),Output:["_col0","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=287989836 width=135) - Output:["_col0","_col1"] - TableScan [TS_40] (rows=287989836 width=135) - default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_59] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_58] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0, _col1 - Group By Operator [GBY_56] (rows=9223372036854775807 width=1) - Output:["_col0","_col1"],keys:_col4, _col3 - Select Operator [SEL_55] (rows=9223372036854775807 width=1) - Output:["_col4","_col3"] - Merge Join Operator [MERGEJOIN_142] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] - <-Map 24 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_50] - Select Operator [SEL_43] (rows=73049 width=4) - TableScan [TS_42] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_51] - Select Operator [SEL_45] (rows=60 width=4) - TableScan [TS_44] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE - <-Map 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Select Operator [SEL_47] (rows=40000000 width=4) - TableScan [TS_46] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Map 29 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1"] - TableScan [TS_48] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col3, _col4 - Merge Join Operator [MERGEJOIN_148] (rows=2790070162094850048 width=1) - Conds:RS_78._col4=RS_79._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_38] (rows=2536427365110644736 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Group By Operator [GBY_36] (rows=5072854730221289472 width=1) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_146] (rows=5072854730221289472 width=1) - Conds:RS_32._col0=RS_33._col0(Inner),Output:["_col0"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=28798881 width=106) - Output:["_col0"] - TableScan [TS_30] (rows=28798881 width=106) - default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_26] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_141] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3"] - <-Map 12 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_20] - Select Operator [SEL_13] (rows=73049 width=4) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_21] - Select Operator [SEL_15] (rows=60 width=4) - TableScan [TS_14] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE - <-Map 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_22] - Select Operator [SEL_17] (rows=40000000 width=4) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Map 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Select Operator [SEL_19] (rows=287989836 width=135) - Output:["_col0"] - TableScan [TS_18] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_order_number"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_78] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_145] (rows=383314495 width=135) - Conds:RS_75._col2=RS_76._col0(Inner),Output:["_col3","_col4","_col5","_col6"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=30 width=2045) - Output:["_col0"] - Filter Operator [FIL_136] (rows=30 width=2045) - predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) - TableScan [TS_9] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_144] (rows=348467716 width=135) - Conds:RS_72._col1=RS_73._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_135] (rows=20000000 width=1014) - predicate:((ca_state = 'NY') and ca_address_sk is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_143] (rows=316788826 width=135) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_133] (rows=287989836 width=135) - predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_134] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-4-01' AND 2001-05-31 01:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query17.q.out ql/src/test/results/clientpositive/perf/query17.q.out index f03aeaa..3d3e265 100644 --- ql/src/test/results/clientpositive/perf/query17.q.out +++ ql/src/test/results/clientpositive/perf/query17.q.out @@ -5,143 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Select Operator [SEL_50] (rows=510205767 width=88) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["count(VALUE._col0)","avg(VALUE._col1)","stddev_samp(VALUE._col2)","count(VALUE._col3)","avg(VALUE._col4)","stddev_samp(VALUE._col5)","count(VALUE._col6)","avg(VALUE._col7)","stddev_samp(VALUE._col8)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["count(_col15)","avg(_col15)","stddev_samp(_col15)","count(_col20)","avg(_col20)","stddev_samp(_col20)","count(_col3)","avg(_col3)","stddev_samp(_col3)"],keys:_col24, _col25, _col7 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col24","_col25","_col7","_col15","_col20","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col11=RS_44._col0(Inner),Output:["_col3","_col7","_col15","_col20","_col24","_col25"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col11 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col12, _col11(Inner),Output:["_col3","_col7","_col11","_col15","_col20"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col12, _col11 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col14","_col5","_col9"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col16"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=36525 width=1119) - predicate:((d_quarter_name) IN ('2000Q1', '2000Q2', '2000Q3') and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=36524 width=1119) - predicate:((d_quarter_name = '2000Q1') and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_return_quantity"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_quantity"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=36525 width=1119) - predicate:((d_quarter_name) IN ('2000Q1', '2000Q2', '2000Q3') and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query18.q.out ql/src/test/results/clientpositive/perf/query18.q.out index cf11954..339f4a7 100644 --- ql/src/test/results/clientpositive/perf/query18.q.out +++ ql/src/test/results/clientpositive/perf/query18.q.out @@ -5,128 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_49] - Limit [LIM_48] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_47] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_46] - Select Operator [SEL_45] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Group By Operator [GBY_44] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)","avg(VALUE._col4)","avg(VALUE._col5)","avg(VALUE._col6)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_42] (rows=2108229765 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["avg(_col4)","avg(_col5)","avg(_col6)","avg(_col7)","avg(_col8)","avg(_col9)","avg(_col10)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_40] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Merge Join Operator [MERGEJOIN_83] (rows=421645953 width=135) - Conds:RS_37._col0=RS_38._col3(Inner),Output:["_col4","_col6","_col7","_col8","_col11","_col16","_col17","_col18","_col19","_col20","_col26"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col3 - Select Operator [SEL_30] (rows=383314495 width=135) - Output:["_col1","_col10","_col16","_col3","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_82] (rows=383314495 width=135) - Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col8","_col14","_col16"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_18] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_81] (rows=348467716 width=135) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col14"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=465450 width=385) - Output:["_col0","_col3"] - Filter Operator [FIL_76] (rows=465450 width=385) - predicate:((cd_gender = 'M') and (cd_education_status = 'College') and cd_demo_sk is not null) - TableScan [TS_15] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_education_status","cd_dep_count"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_80] (rows=316788826 width=135) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_75] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Filter Operator [FIL_74] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_cdemo_sk is not null and cs_item_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_9] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_bill_cdemo_sk","cs_item_sk","cs_quantity","cs_list_price","cs_sales_price","cs_coupon_amt","cs_net_profit"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_79] (rows=48400001 width=860) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0","_col4","_col6","_col7","_col8"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_73] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_6] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_78] (rows=44000000 width=860) - Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col0","_col1","_col4","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=40000000 width=860) - Output:["_col0","_col1","_col2","_col4"] - Filter Operator [FIL_71] (rows=40000000 width=860) - predicate:((c_birth_month) IN (9, 5, 12, 4, 1, 10) and c_customer_sk is not null and c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk","c_birth_month","c_birth_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_72] (rows=20000000 width=1014) - predicate:((ca_state) IN ('ND', 'WI', 'AL', 'NC', 'OK', 'MS', 'TN') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_state","ca_country"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query19.q.out ql/src/test/results/clientpositive/perf/query19.q.out index 0ddcd83..9f445cc 100644 --- ql/src/test/results/clientpositive/perf/query19.q.out +++ ql/src/test/results/clientpositive/perf/query19.q.out @@ -5,117 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_45] - Limit [LIM_44] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_43] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - Select Operator [SEL_40] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_39] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_37] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col7)"],keys:_col10, _col9, _col11, _col12 - Select Operator [SEL_36] (rows=843315281 width=88) - Output:["_col10","_col9","_col11","_col12","_col7"] - Filter Operator [FIL_35] (rows=843315281 width=88) - predicate:(substr(_col17, 1, 5) <> substr(_col19, 1, 5)) - Select Operator [SEL_34] (rows=843315281 width=88) - Output:["_col7","_col9","_col10","_col11","_col12","_col17","_col19"] - Merge Join Operator [MERGEJOIN_73] (rows=843315281 width=88) - Conds:RS_31._col0=RS_32._col2(Inner),Output:["_col3","_col8","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Select Operator [SEL_27] (rows=766650239 width=88) - Output:["_col10","_col11","_col12","_col15","_col2","_col4","_col9"] - Merge Join Operator [MERGEJOIN_72] (rows=766650239 width=88) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col2","_col4","_col9","_col10","_col11","_col12","_col15"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_68] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_zip"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_71] (rows=696954748 width=88) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col3","_col4","_col9","_col10","_col11","_col12"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_67] (rows=231000 width=1436) - predicate:((i_manager_id = 7) and i_item_sk is not null) - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manufact_id","i_manufact","i_manager_id"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_70] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_66] (rows=18262 width=1119) - predicate:((d_moy = 11) and (d_year = 1999) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_65] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_customer_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_69] (rows=88000001 width=860) - Conds:RS_28._col1=RS_29._col0(Inner),Output:["_col0","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_63] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_64] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_zip"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query20.q.out ql/src/test/results/clientpositive/perf/query20.q.out index c525703..3ba37dd 100644 --- ql/src/test/results/clientpositive/perf/query20.q.out +++ ql/src/test/results/clientpositive/perf/query20.q.out @@ -5,76 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_26] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_23] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_22] (rows=174233858 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_21] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col3 - Select Operator [SEL_19] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_18] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_16] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)"],keys:_col10, _col9, _col6, _col7, _col8 - Select Operator [SEL_15] (rows=348467716 width=135) - Output:["_col10","_col9","_col6","_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_38] (rows=348467716 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_36] (rows=231000 width=1436) - predicate:((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_37] (rows=316788826 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_34] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_35] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-01-12' AND '2001-02-11' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query21.q.out ql/src/test/results/clientpositive/perf/query21.q.out index 67a49e2..75de1c4 100644 --- ql/src/test/results/clientpositive/perf/query21.q.out +++ ql/src/test/results/clientpositive/perf/query21.q.out @@ -59,81 +59,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_31] - Limit [LIM_30] (rows=100 width=15) - Number of rows:100 - Select Operator [SEL_29] (rows=12506076 width=15) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col2 > 0)) THEN ((UDFToDouble(_col3) / UDFToDouble(_col2)) BETWEEN 0.6666666666666666 AND 1.5) ELSE (null) END - Group By Operator [GBY_25] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1 - Group By Operator [GBY_23] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_47] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col7","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_44] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_46] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=51333 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_43] (rows=51333 width=1436) - predicate:(i_current_price BETWEEN 0.99 AND 1.49 and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_current_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_45] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_41] (rows=37584000 width=15) - predicate:(inv_warehouse_sk is not null and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-03-08 23:00:00.0 AND 1998-05-08 00:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query22.q.out ql/src/test/results/clientpositive/perf/query22.q.out index 6ff8896..03ceb09 100644 --- ql/src/test/results/clientpositive/perf/query22.q.out +++ ql/src/test/results/clientpositive/perf/query22.q.out @@ -43,81 +43,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_30] - Limit [LIM_29] (rows=100 width=15) - Number of rows:100 - Select Operator [SEL_28] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - Select Operator [SEL_25] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col5"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_22] (rows=250121525 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(_col3)"],keys:_col8, _col9, _col10, _col11, 0 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col8","_col9","_col10","_col11","_col3"] - Merge Join Operator [MERGEJOIN_46] (rows=50024305 width=15) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col8","_col9","_col10","_col11"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_43] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_45] (rows=45476640 width=15) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=27 width=1029) - Output:["_col0"] - Filter Operator [FIL_42] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_6] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_44] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=37584000 width=15) - predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_41] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query23.q.out ql/src/test/results/clientpositive/perf/query23.q.out index 59f2586..7b350c4 100644 --- ql/src/test/results/clientpositive/perf/query23.q.out +++ ql/src/test/results/clientpositive/perf/query23.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join MERGEJOIN[382][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 51' is a cross product -Warning: Shuffle Join MERGEJOIN[383][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 52' is a cross product -Warning: Shuffle Join MERGEJOIN[379][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 19' is a cross product -Warning: Shuffle Join MERGEJOIN[380][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 20' is a cross product +Warning: Shuffle Join MERGEJOIN[379][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 19' is a cross product +Warning: Shuffle Join MERGEJOIN[380][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 20' is a cross product +Warning: Shuffle Join MERGEJOIN[382][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 51' is a cross product +Warning: Shuffle Join MERGEJOIN[383][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 52' is a cross product PREHOOK: query: explain with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales @@ -105,553 +105,49 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 16 <- Map 23 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (CUSTOM_SIMPLE_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (CUSTOM_SIMPLE_EDGE), Reducer 33 (CUSTOM_SIMPLE_EDGE) -Reducer 21 <- Reducer 20 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 29 (SIMPLE_EDGE) -Reducer 26 <- Map 30 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 32 <- Map 31 (SIMPLE_EDGE), Map 34 (SIMPLE_EDGE) -Reducer 33 <- Reducer 32 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Reducer 36 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 4 <- Reducer 21 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 41 <- Map 40 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE) -Reducer 42 <- Map 45 (SIMPLE_EDGE), Reducer 41 (SIMPLE_EDGE) -Reducer 43 <- Reducer 42 (SIMPLE_EDGE) -Reducer 47 <- Map 46 (SIMPLE_EDGE), Map 54 (SIMPLE_EDGE) -Reducer 48 <- Map 55 (SIMPLE_EDGE), Reducer 47 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (CUSTOM_SIMPLE_EDGE) -Reducer 51 <- Reducer 50 (CUSTOM_SIMPLE_EDGE), Reducer 60 (CUSTOM_SIMPLE_EDGE) -Reducer 52 <- Reducer 51 (CUSTOM_SIMPLE_EDGE), Reducer 65 (CUSTOM_SIMPLE_EDGE) -Reducer 53 <- Reducer 52 (SIMPLE_EDGE) -Reducer 57 <- Map 56 (SIMPLE_EDGE), Map 61 (SIMPLE_EDGE) -Reducer 58 <- Map 62 (SIMPLE_EDGE), Reducer 57 (SIMPLE_EDGE) -Reducer 59 <- Reducer 58 (SIMPLE_EDGE) -Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) -Reducer 60 <- Reducer 59 (CUSTOM_SIMPLE_EDGE) -Reducer 64 <- Map 63 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE) -Reducer 65 <- Reducer 64 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (PARTITION_EDGE) +Vertex 19 <- Vertex 18 (PARTITION_EDGE), Vertex 28 (PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (PARTITION_EDGE), Vertex 33 (PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 43 (SORT_PARTITION_EDGE) +Vertex 38 <- Union 5 (CONTAINS), Vertex 37 (SORT_PARTITION_EDGE), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 41 <- Vertex 40 (SORT_PARTITION_EDGE), Vertex 44 (SORT_PARTITION_EDGE) +Vertex 42 <- Vertex 41 (SORT_PARTITION_EDGE), Vertex 45 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE) +Vertex 47 <- Vertex 46 (SORT_PARTITION_EDGE), Vertex 54 (SORT_PARTITION_EDGE) +Vertex 48 <- Vertex 47 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (PARTITION_EDGE) +Vertex 51 <- Vertex 50 (PARTITION_EDGE), Vertex 60 (PARTITION_EDGE) +Vertex 52 <- Vertex 51 (PARTITION_EDGE), Vertex 65 (PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE) +Vertex 57 <- Vertex 56 (SORT_PARTITION_EDGE), Vertex 61 (SORT_PARTITION_EDGE) +Vertex 58 <- Vertex 57 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (PARTITION_EDGE) +Vertex 60 <- Vertex 59 (PARTITION_EDGE) +Vertex 64 <- Vertex 63 (SORT_PARTITION_EDGE), Vertex 66 (SORT_PARTITION_EDGE) +Vertex 65 <- Vertex 64 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_258] - Limit [LIM_257] (rows=1 width=112) - Number of rows:100 - Group By Operator [GBY_255] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Union 5 [CUSTOM_SIMPLE_EDGE] - <-Reducer 38 [CONTAINS] - Reduce Output Operator [RS_254] - Group By Operator [GBY_253] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col0)"] - Select Operator [SEL_249] (rows=191667562 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_384] (rows=191667562 width=135) - Conds:RS_246._col2=RS_247._col0(Inner),Output:["_col3","_col4"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_246] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_378] (rows=174243235 width=135) - Conds:RS_243._col1=RS_244._col0(Inner),Output:["_col2","_col3","_col4"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_243] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_369] (rows=158402938 width=135) - Conds:RS_240._col0=RS_241._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_240] - PartitionCols:_col0 - Select Operator [SEL_127] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_346] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_bill_customer_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_125] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_quantity","ws_list_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_241] - PartitionCols:_col0 - Select Operator [SEL_130] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_347] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_128] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_244] - PartitionCols:_col0 - Group By Operator [GBY_156] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_152] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_151] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_360] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_150] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 42 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0 - Group By Operator [GBY_148] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_146] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_371] (rows=696954748 width=88) - Conds:RS_143._col1=RS_144._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 45 [SIMPLE_EDGE] - SHUFFLE [RS_144] - PartitionCols:_col0 - Select Operator [SEL_139] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_350] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_137] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 41 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_370] (rows=633595212 width=88) - Conds:RS_140._col0=RS_141._col0(Inner),Output:["_col1","_col3"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_140] - PartitionCols:_col0 - Select Operator [SEL_133] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_348] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_131] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 44 [SIMPLE_EDGE] - SHUFFLE [RS_141] - PartitionCols:_col0 - Select Operator [SEL_136] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_349] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_134] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_247] - PartitionCols:_col0 - Group By Operator [GBY_238] (rows=52799601 width=322) - Output:["_col0"],keys:KEY._col0 - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_237] - PartitionCols:_col0 - Group By Operator [GBY_236] (rows=105599202 width=322) - Output:["_col0"],keys:_col2 - Select Operator [SEL_235] (rows=105599202 width=322) - Output:["_col2"] - Filter Operator [FIL_234] (rows=105599202 width=322) - predicate:(_col3 > (0.95 * _col1)) - Merge Join Operator [MERGEJOIN_383] (rows=316797606 width=322) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 51 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_231] - Merge Join Operator [MERGEJOIN_382] (rows=1 width=233) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 50 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_228] - Select Operator [SEL_186] (rows=1 width=8) - Filter Operator [FIL_185] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_183] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 49 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_182] - Group By Operator [GBY_181] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_178] (rows=348477374 width=88) - Group By Operator [GBY_177] (rows=348477374 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_176] - PartitionCols:_col0 - Group By Operator [GBY_175] (rows=696954748 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_173] (rows=696954748 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_373] (rows=696954748 width=88) - Conds:RS_170._col1=RS_171._col0(Inner),Output:["_col6"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col0 - Select Operator [SEL_166] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_353] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_164] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 47 [SIMPLE_EDGE] - SHUFFLE [RS_170] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_372] (rows=633595212 width=88) - Conds:RS_167._col0=RS_168._col0(Inner),Output:["_col1"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_167] - PartitionCols:_col0 - Select Operator [SEL_160] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_351] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_158] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 54 [SIMPLE_EDGE] - SHUFFLE [RS_168] - PartitionCols:_col0 - Select Operator [SEL_163] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_352] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_161] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 60 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_229] - Group By Operator [GBY_211] (rows=1 width=224) - Output:["_col0"],aggregations:["max(VALUE._col0)"] - <-Reducer 59 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_210] - Group By Operator [GBY_209] (rows=1 width=224) - Output:["_col0"],aggregations:["max(_col1)"] - Select Operator [SEL_207] (rows=348477374 width=88) - Output:["_col1"] - Group By Operator [GBY_206] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 58 [SIMPLE_EDGE] - SHUFFLE [RS_205] - PartitionCols:_col0 - Group By Operator [GBY_204] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_202] (rows=696954748 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_375] (rows=696954748 width=88) - Conds:RS_199._col1=RS_200._col0(Inner),Output:["_col2","_col3","_col6"] - <-Map 62 [SIMPLE_EDGE] - SHUFFLE [RS_200] - PartitionCols:_col0 - Select Operator [SEL_195] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_356] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_193] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 57 [SIMPLE_EDGE] - SHUFFLE [RS_199] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_374] (rows=633595212 width=88) - Conds:RS_196._col0=RS_197._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_196] - PartitionCols:_col0 - Select Operator [SEL_189] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_354] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_187] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 61 [SIMPLE_EDGE] - SHUFFLE [RS_197] - PartitionCols:_col0 - Select Operator [SEL_192] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_355] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_190] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_232] - Group By Operator [GBY_226] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 64 [SIMPLE_EDGE] - SHUFFLE [RS_225] - PartitionCols:_col0 - Group By Operator [GBY_224] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_222] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_376] (rows=633595212 width=88) - Conds:RS_219._col0=RS_220._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_219] - PartitionCols:_col0 - Select Operator [SEL_215] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_357] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - TableScan [TS_213] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 66 [SIMPLE_EDGE] - SHUFFLE [RS_220] - PartitionCols:_col0 - Select Operator [SEL_218] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_358] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_216] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_254] - Group By Operator [GBY_253] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col0)"] - Select Operator [SEL_124] (rows=383314495 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_381] (rows=383314495 width=135) - Conds:RS_121._col1=RS_122._col0(Inner),Output:["_col3","_col4"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Group By Operator [GBY_113] (rows=52799601 width=322) - Output:["_col0"],keys:KEY._col0 - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Group By Operator [GBY_111] (rows=105599202 width=322) - Output:["_col0"],keys:_col2 - Select Operator [SEL_110] (rows=105599202 width=322) - Output:["_col2"] - Filter Operator [FIL_109] (rows=105599202 width=322) - predicate:(_col3 > (0.95 * _col1)) - Merge Join Operator [MERGEJOIN_380] (rows=316797606 width=322) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_106] - Merge Join Operator [MERGEJOIN_379] (rows=1 width=233) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_103] - Select Operator [SEL_61] (rows=1 width=8) - Filter Operator [FIL_60] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_58] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_57] - Group By Operator [GBY_56] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_53] (rows=348477374 width=88) - Group By Operator [GBY_52] (rows=348477374 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_50] (rows=696954748 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_48] (rows=696954748 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_365] (rows=696954748 width=88) - Conds:RS_45._col1=RS_46._col0(Inner),Output:["_col6"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_340] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_39] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_364] (rows=633595212 width=88) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_338] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_33] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_339] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_104] - Group By Operator [GBY_86] (rows=1 width=224) - Output:["_col0"],aggregations:["max(VALUE._col0)"] - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_85] - Group By Operator [GBY_84] (rows=1 width=224) - Output:["_col0"],aggregations:["max(_col1)"] - Select Operator [SEL_82] (rows=348477374 width=88) - Output:["_col1"] - Group By Operator [GBY_81] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_79] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_77] (rows=696954748 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_367] (rows=696954748 width=88) - Conds:RS_74._col1=RS_75._col0(Inner),Output:["_col2","_col3","_col6"] - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Select Operator [SEL_70] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_343] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_68] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_366] (rows=633595212 width=88) - Conds:RS_71._col0=RS_72._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Select Operator [SEL_64] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_341] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_62] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col0 - Select Operator [SEL_67] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_342] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_65] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 33 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_107] - Group By Operator [GBY_101] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_99] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_97] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_368] (rows=633595212 width=88) - Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_90] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_344] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - TableScan [TS_88] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_93] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_345] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_91] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_377] (rows=348467716 width=135) - Conds:RS_118._col2=RS_119._col0(Inner),Output:["_col1","_col3","_col4"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_27] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_26] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_359] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_25] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_21] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_363] (rows=696954748 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_337] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_362] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_336] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_335] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_361] (rows=316788826 width=135) - Conds:RS_115._col0=RS_116._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_333] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_bill_customer_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity","cs_list_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_334] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query25.q.out ql/src/test/results/clientpositive/perf/query25.q.out index 8e5119c..fa6b185 100644 --- ql/src/test/results/clientpositive/perf/query25.q.out +++ ql/src/test/results/clientpositive/perf/query25.q.out @@ -5,141 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col18)","sum(_col23)","sum(_col3)"],keys:_col28, _col29, _col8, _col9 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col28","_col29","_col8","_col9","_col18","_col23","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col14=RS_44._col0(Inner),Output:["_col3","_col8","_col9","_col18","_col23","_col28","_col29"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col14 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col14, _col13(Inner),Output:["_col3","_col8","_col9","_col14","_col18","_col23"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col14, _col13 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col13","_col14","_col16","_col2","_col7"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col18","_col19"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_moy = 4) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_net_loss"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_net_profit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_net_profit"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query26.q.out ql/src/test/results/clientpositive/perf/query26.q.out index 6f7aaf7..73205b6 100644 --- ql/src/test/results/clientpositive/perf/query26.q.out +++ ql/src/test/results/clientpositive/perf/query26.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_33] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_30] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(_col4)","avg(_col5)","avg(_col7)","avg(_col6)"],keys:_col18 - Select Operator [SEL_27] (rows=421645953 width=135) - Output:["_col18","_col4","_col5","_col7","_col6"] - Merge Join Operator [MERGEJOIN_58] (rows=421645953 width=135) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col18"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=383314495 width=135) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col2","_col4","_col5","_col6","_col7"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_53] (rows=2300 width=1179) - predicate:(((p_channel_email = 'N') or (p_channel_event = 'N')) and p_promo_sk is not null) - TableScan [TS_9] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_email","p_channel_event"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_56] (rows=348467716 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_52] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=316788826 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_50] (rows=287989836 width=135) - predicate:(cs_bill_cdemo_sk is not null and cs_sold_date_sk is not null and cs_item_sk is not null and cs_promo_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_cdemo_sk","cs_item_sk","cs_promo_sk","cs_quantity","cs_list_price","cs_sales_price","cs_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_51] (rows=232725 width=385) - predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query27.q.out ql/src/test/results/clientpositive/perf/query27.q.out index cbd7d29..fe38aa9 100644 --- ql/src/test/results/clientpositive/perf/query27.q.out +++ ql/src/test/results/clientpositive/perf/query27.q.out @@ -45,96 +45,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Select Operator [SEL_32] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_31] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_29] (rows=2529945843 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["avg(_col2)","avg(_col3)","avg(_col4)","avg(_col5)"],keys:_col0, _col1, 0 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_59] (rows=843315281 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col15","_col17"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_55] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_58] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col15"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=852 width=1910) - predicate:((s_state) IN ('SD', 'FL', 'MI', 'LA', 'MO', 'SC') and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_57] (rows=696954748 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_53] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_56] (rows=633595212 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_51] (rows=575995635 width=88) - predicate:(ss_cdemo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_cdemo_sk","ss_store_sk","ss_quantity","ss_list_price","ss_sales_price","ss_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_52] (rows=232725 width=385) - predicate:((cd_gender = 'M') and (cd_marital_status = 'U') and (cd_education_status = '2 yr Degree') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query28.q.out ql/src/test/results/clientpositive/perf/query28.q.out index f7c5225..9c6aa18 100644 --- ql/src/test/results/clientpositive/perf/query28.q.out +++ ql/src/test/results/clientpositive/perf/query28.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT @@ -104,108 +104,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (PARTITION_EDGE), Vertex 13 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_51] - Limit [LIM_50] (rows=1 width=2497) - Number of rows:100 - Select Operator [SEL_49] (rows=1 width=2497) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Merge Join Operator [MERGEJOIN_58] (rows=1 width=2497) - Conds:(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - <-Reducer 11 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_46] - Group By Operator [GBY_33] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_31] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_30] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_56] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 11 AND 15 and (ss_list_price BETWEEN 66 AND 76 or ss_coupon_amt BETWEEN 920 AND 1920 or ss_wholesale_cost BETWEEN 4 AND 24)) - TableScan [TS_28] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 13 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_47] - Group By Operator [GBY_40] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_39] - Group By Operator [GBY_38] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_37] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_57] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 6 AND 10 and (ss_list_price BETWEEN 91 AND 101 or ss_coupon_amt BETWEEN 1430 AND 2430 or ss_wholesale_cost BETWEEN 32 AND 52)) - TableScan [TS_35] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_42] - Group By Operator [GBY_5] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_2] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_52] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 0 AND 5 and (ss_list_price BETWEEN 11 AND 21 or ss_coupon_amt BETWEEN 460 AND 1460 or ss_wholesale_cost BETWEEN 14 AND 34)) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_43] - Group By Operator [GBY_12] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_9] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_53] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 26 AND 30 and (ss_list_price BETWEEN 28 AND 38 or ss_coupon_amt BETWEEN 2513 AND 3513 or ss_wholesale_cost BETWEEN 42 AND 62)) - TableScan [TS_7] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 7 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_44] - Group By Operator [GBY_19] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_18] - Group By Operator [GBY_17] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_16] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_54] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 21 AND 25 and (ss_list_price BETWEEN 135 AND 145 or ss_coupon_amt BETWEEN 14180 AND 15180 or ss_wholesale_cost BETWEEN 38 AND 58)) - TableScan [TS_14] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 9 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_45] - Group By Operator [GBY_26] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_24] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_23] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_55] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 16 AND 20 and (ss_list_price BETWEEN 142 AND 152 or ss_coupon_amt BETWEEN 3054 AND 4054 or ss_wholesale_cost BETWEEN 80 AND 100)) - TableScan [TS_21] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query29.q.out ql/src/test/results/clientpositive/perf/query29.q.out index 587c027..f1fe37a 100644 --- ql/src/test/results/clientpositive/perf/query29.q.out +++ ql/src/test/results/clientpositive/perf/query29.q.out @@ -5,141 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col17)","sum(_col22)","sum(_col3)"],keys:_col27, _col28, _col7, _col8 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col27","_col28","_col7","_col8","_col17","_col22","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col13=RS_44._col0(Inner),Output:["_col3","_col7","_col8","_col17","_col22","_col27","_col28"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col13 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col14, _col13(Inner),Output:["_col3","_col7","_col8","_col13","_col17","_col22"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col14, _col13 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col13","_col14","_col16","_col2","_col7"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col18","_col19"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 2 AND 5 and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_moy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_return_quantity"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_quantity"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=36525 width=1119) - predicate:((d_year) IN (2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query3.q.out ql/src/test/results/clientpositive/perf/query3.q.out index 2845a88..acddbe0 100644 --- ql/src/test/results/clientpositive/perf/query3.q.out +++ ql/src/test/results/clientpositive/perf/query3.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col8, _col4, _col5 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col8","_col4","_col5","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col2","_col4","_col5","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=36524 width=1119) - predicate:((d_moy = 12) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=231000 width=1436) - predicate:((i_manufact_id = 436) and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manufact_id"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query30.q.out ql/src/test/results/clientpositive/perf/query30.q.out index 8d38fe2..ca187c4 100644 --- ql/src/test/results/clientpositive/perf/query30.q.out +++ ql/src/test/results/clientpositive/perf/query30.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 22' is a cross product +Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 22' is a cross product PREHOOK: query: explain with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, @@ -60,249 +60,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE), Map 30 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 26 <- Map 29 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_102] - Limit [LIM_101] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_100] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_99] - Select Operator [SEL_98] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Filter Operator [FIL_97] (rows=930023387364950016 width=1) - predicate:(_col2 > CASE WHEN (_col20 is null) THEN (null) ELSE (_col19) END) - Select Operator [SEL_96] (rows=2790070162094850048 width=1) - Output:["_col2","_col6","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - Merge Join Operator [MERGEJOIN_162] (rows=2790070162094850048 width=1) - Conds:RS_93._col17=RS_94._col2(Left Outer),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col18","_col19","_col20"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col2 - Select Operator [SEL_86] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_85] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col3 - Select Operator [SEL_82] (rows=5072854730221289472 width=1) - Output:["_col3","_col2"] - Merge Join Operator [MERGEJOIN_161] (rows=5072854730221289472 width=1) - Conds:RS_79._col1=RS_80._col0(Inner),Output:["_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=22000000 width=1014) - Output:["_col1","_col2"] - Group By Operator [GBY_44] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1 - Group By Operator [GBY_42] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col7, _col1 - Select Operator [SEL_41] (rows=44000000 width=1014) - Output:["_col7","_col1","_col3"] - Merge Join Operator [MERGEJOIN_159] (rows=44000000 width=1014) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_32] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_158] (rows=15838314 width=92) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_146] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_returning_addr_sk is not null) - TableScan [TS_26] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_returning_customer_sk","wr_returning_addr_sk","wr_return_amt"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_147] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_77] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Group By Operator [GBY_75] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_154] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_70] - Select Operator [SEL_47] (rows=80000000 width=4) - TableScan [TS_46] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:COMPLETE - <-Map 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_72] - Select Operator [SEL_69] (rows=40000000 width=4) - TableScan [TS_68] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_67] (rows=22000000 width=1014) - Output:["_col1"] - Group By Operator [GBY_66] (rows=22000000 width=1014) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Group By Operator [GBY_64] (rows=44000000 width=1014) - Output:["_col0","_col1"],keys:_col7, _col1 - Select Operator [SEL_63] (rows=44000000 width=1014) - Output:["_col7","_col1"] - Merge Join Operator [MERGEJOIN_153] (rows=44000000 width=1014) - Conds:RS_60._col2=RS_61._col0(Inner),Output:["_col1","_col7"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_54] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_152] (rows=15838314 width=92) - Conds:RS_57._col0=RS_58._col0(Inner),Output:["_col1","_col2"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=14398467 width=92) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_149] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_returning_addr_sk is not null) - TableScan [TS_48] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_returning_customer_sk","wr_returning_addr_sk"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_150] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_51] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col17 - Merge Join Operator [MERGEJOIN_160] (rows=96800003 width=860) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col17","_col18"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1 - Group By Operator [GBY_22] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col1, _col7 - Select Operator [SEL_21] (rows=44000000 width=1014) - Output:["_col1","_col7","_col3"] - Merge Join Operator [MERGEJOIN_157] (rows=44000000 width=1014) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_12] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_156] (rows=15838314 width=92) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_144] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_143] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_returning_addr_sk is not null and wr_returning_customer_sk is not null) - TableScan [TS_6] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_returning_customer_sk","wr_returning_addr_sk","wr_return_amt"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_155] (rows=88000001 width=860) - Conds:RS_87._col2=RS_88._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"] - Filter Operator [FIL_141] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_current_addr_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_day","c_birth_month","c_birth_year","c_birth_country","c_login","c_email_address","c_last_review_date"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_142] (rows=20000000 width=1014) - predicate:((ca_state = 'IL') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query31.q.out ql/src/test/results/clientpositive/perf/query31.q.out index 433d419..4ea7e20 100644 --- ql/src/test/results/clientpositive/perf/query31.q.out +++ ql/src/test/results/clientpositive/perf/query31.q.out @@ -5,333 +5,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 24 <- Map 28 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 25 <- Reducer 24 (SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 33 (SIMPLE_EDGE) -Reducer 31 <- Map 34 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Reducer 31 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Map 40 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 38 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 38 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_140] - Select Operator [SEL_139] (rows=316243230 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_138] - Select Operator [SEL_137] (rows=316243230 width=88) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_136] (rows=316243230 width=88) - predicate:CASE WHEN ((_col3 > 0)) THEN (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > (_col5 / _col3))) ELSE ((null > (_col5 / _col3))) END) ELSE (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > null)) ELSE (null) END) END - Merge Join Operator [MERGEJOIN_272] (rows=632486460 width=88) - Conds:RS_132._col6=RS_133._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col7","_col9","_col11"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Group By Operator [GBY_130] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col0 - Group By Operator [GBY_128] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_127] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_269] (rows=174243235 width=135) - Conds:RS_124._col1=RS_125._col0(Inner),Output:["_col2","_col7"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0 - Select Operator [SEL_120] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_257] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_118] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_124] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_268] (rows=158402938 width=135) - Conds:RS_121._col0=RS_122._col0(Inner),Output:["_col1","_col2"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col0 - Select Operator [SEL_114] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_255] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_112] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Select Operator [SEL_117] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_256] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_115] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col6 - Filter Operator [FIL_110] (rows=574987679 width=88) - predicate:CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > (_col3 / _col1))) ELSE ((null > (_col3 / _col1))) END) ELSE (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > null)) ELSE (null) END) END - Select Operator [SEL_109] (rows=1149975359 width=88) - Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_271] (rows=1149975359 width=88) - Conds:RS_104._col0=RS_105._col0(Inner),RS_104._col0=RS_106._col0(Inner),RS_104._col0=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Group By Operator [GBY_36] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_35] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_261] (rows=696954748 width=88) - Conds:RS_32._col1=RS_33._col0(Inner),Output:["_col2","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_245] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_26] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_260] (rows=633595212 width=88) - Conds:RS_29._col0=RS_30._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_243] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_20] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_244] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col0 - Group By Operator [GBY_58] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Group By Operator [GBY_56] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_55] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_263] (rows=696954748 width=88) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_248] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_46] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_262] (rows=633595212 width=88) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_246] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_40] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_45] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_247] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_43] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_270] (rows=95833780 width=135) - Conds:RS_100._col0=RS_101._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_78] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Group By Operator [GBY_76] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_75] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_265] (rows=174243235 width=135) - Conds:RS_72._col1=RS_73._col0(Inner),Output:["_col2","_col7"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_68] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_251] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_66] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_264] (rows=158402938 width=135) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_62] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_249] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_60] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_65] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_250] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_63] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_98] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Group By Operator [GBY_96] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_95] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_267] (rows=174243235 width=135) - Conds:RS_92._col1=RS_93._col0(Inner),Output:["_col2","_col7"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_254] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_86] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_266] (rows=158402938 width=135) - Conds:RS_89._col0=RS_90._col0(Inner),Output:["_col1","_col2"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_252] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_80] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_85] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_253] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_83] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_259] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_242] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_258] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_240] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_241] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query32.q.out ql/src/test/results/clientpositive/perf/query32.q.out index 6311054..5304734 100644 --- ql/src/test/results/clientpositive/perf/query32.q.out +++ ql/src/test/results/clientpositive/perf/query32.q.out @@ -39,96 +39,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Map 10 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 9 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_38] - Group By Operator [GBY_36] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_35] - Group By Operator [GBY_34] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col1)"] - Select Operator [SEL_33] (rows=232311810 width=135) - Output:["_col1"] - Filter Operator [FIL_32] (rows=232311810 width=135) - predicate:(_col1 > _col5) - Select Operator [SEL_31] (rows=696935432 width=135) - Output:["_col1","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=696935432 width=135) - Conds:RS_27._col0=RS_28._col0(Inner),RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col5"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=231000 width=1436) - Output:["_col0"] - Filter Operator [FIL_55] (rows=231000 width=1436) - predicate:((i_manufact_id = 436) and i_item_sk is not null) - TableScan [TS_14] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=158394413 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_12] (rows=158394413 width=135) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=316788826 width=135) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Merge Join Operator [MERGEJOIN_58] (rows=316788826 width=135) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_53] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2000-01-27' AND '2000-04-27' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_59] (rows=316788826 width=135) - Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_57] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2000-01-27' AND '2000-04-27' and d_date_sk is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_56] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null) - TableScan [TS_17] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query33.q.out ql/src/test/results/clientpositive/perf/query33.q.out index 342bd90..e6cb0b3 100644 --- ql/src/test/results/clientpositive/perf/query33.q.out +++ ql/src/test/results/clientpositive/perf/query33.q.out @@ -147,289 +147,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query34.q.out ql/src/test/results/clientpositive/perf/query34.q.out index 51710bd..8200edd 100644 --- ql/src/test/results/clientpositive/perf/query34.q.out +++ ql/src/test/results/clientpositive/perf/query34.q.out @@ -5,96 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=88000001 width=860) - Conds:RS_31._col0=RS_32._col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Filter Operator [FIL_29] (rows=42591679 width=88) - predicate:_col2 BETWEEN 15 AND 20 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_56] (rows=852 width=1910) - predicate:((s_county) IN ('Kittitas County', 'Adams County', 'Richland County', 'Furnas County', 'Orange County', 'Appanoose County', 'Franklin Parish', 'Tehama County') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1200 width=107) - Output:["_col0"] - Filter Operator [FIL_55] (rows=1200 width=107) - predicate:(((hd_buy_potential = '1001-5000') or (hd_buy_potential = '5001-10000')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.2)) ELSE (null) END and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential","hd_dep_count","hd_vehicle_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_store_sk","ss_ticket_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=8116 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and (d_dom BETWEEN 1 AND 3 or d_dom BETWEEN 25 AND 28) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query36.q.out ql/src/test/results/clientpositive/perf/query36.q.out index b356628..df4c901 100644 --- ql/src/test/results/clientpositive/perf/query36.q.out +++ ql/src/test/results/clientpositive/perf/query36.q.out @@ -59,91 +59,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_33] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Select Operator [SEL_30] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_29] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col4 / _col5) ASC NULLS FIRST","partition by:":"(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((UDFToInteger(grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_28] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((UDFToInteger(grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_26] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5","_col6"] - Group By Operator [GBY_25] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_23] (rows=2299950717 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1, 0 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_51] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col4","_col10","_col11"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_48] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_50] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_47] (rows=852 width=1910) - predicate:((s_state) IN ('SD', 'FL', 'MI', 'LA', 'MO', 'SC', 'AL', 'GA') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_49] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_45] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_ext_sales_price","ss_net_profit"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=36524 width=1119) - predicate:((d_year = 1999) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query37.q.out ql/src/test/results/clientpositive/perf/query37.q.out index 1b61198..bca5d0e 100644 --- ql/src/test/results/clientpositive/perf/query37.q.out +++ ql/src/test/results/clientpositive/perf/query37.q.out @@ -33,71 +33,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_26] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_23] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_21] (rows=633577652 width=135) - Output:["_col0","_col1","_col2"],keys:_col2, _col3, _col4 - Merge Join Operator [MERGEJOIN_44] (rows=633577652 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),RS_17._col0=RS_18._col1(Inner),Output:["_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0"] - Filter Operator [FIL_39] (rows=287989836 width=135) - predicate:cs_item_sk is not null - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25666 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=25666 width=1436) - predicate:((i_manufact_id) IN (678, 964, 918, 849) and i_current_price BETWEEN 22 AND 52 and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_manufact_id"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=4593600 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4176000 width=15) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=4176000 width=15) - predicate:(inv_quantity_on_hand BETWEEN 100 AND 500 and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_6] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2001-06-02 AND 2001-08-01 00:00:00.0 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query38.q.out ql/src/test/results/clientpositive/perf/query38.q.out index 1db8b6a..476830b 100644 --- ql/src/test/results/clientpositive/perf/query38.q.out +++ ql/src/test/results/clientpositive/perf/query38.q.out @@ -45,188 +45,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 5 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 5 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_92] - Limit [LIM_91] (rows=1 width=16) - Number of rows:100 - Group By Operator [GBY_89] (rows=1 width=16) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_88] - Group By Operator [GBY_87] (rows=1 width=16) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_85] (rows=1 width=108) - Filter Operator [FIL_84] (rows=1 width=108) - predicate:(_col3 = 3) - Select Operator [SEL_117] (rows=152458212 width=108) - Output:["_col3"] - Group By Operator [GBY_83] (rows=152458212 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_50] (rows=87116929 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_44] (rows=174233858 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_42] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_41] (rows=348467716 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_121] (rows=348467716 width=135) - Conds:RS_38._col1=RS_39._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_113] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_32] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_120] (rows=316788826 width=135) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_111] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_26] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_112] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_77] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_71] (rows=87121617 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_69] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_68] (rows=174243235 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_123] (rows=174243235 width=135) - Conds:RS_65._col1=RS_66._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Select Operator [SEL_61] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_116] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_59] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_122] (rows=158402938 width=135) - Conds:RS_62._col0=RS_63._col0(Inner),Output:["_col1","_col3"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_114] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_53] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Select Operator [SEL_58] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_115] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_56] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_24] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_119] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_110] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_118] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_108] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_customer_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_109] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query39.q.out ql/src/test/results/clientpositive/perf/query39.q.out index bde20a6..3b83861 100644 --- ql/src/test/results/clientpositive/perf/query39.q.out +++ ql/src/test/results/clientpositive/perf/query39.q.out @@ -5,163 +5,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_62] - Select Operator [SEL_61] (rows=13756683 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Select Operator [SEL_59] (rows=13756683 width=15) - Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col8","_col9"] - Merge Join Operator [MERGEJOIN_103] (rows=13756683 width=15) - Conds:RS_56._col1, _col2=RS_57._col1, _col2(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col1, _col2 - Select Operator [SEL_55] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_54] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_53] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_52] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_50] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Select Operator [SEL_49] (rows=50024305 width=15) - Output:["_col8","_col7","_col9","_col3"] - Merge Join Operator [MERGEJOIN_102] (rows=50024305 width=15) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_37] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_101] (rows=45476640 width=15) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_95] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_100] (rows=41342400 width=15) - Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_30] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_93] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_28] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_33] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_94] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 4) and d_date_sk is not null) - TableScan [TS_31] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col1, _col2 - Select Operator [SEL_27] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_25] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_22] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col8","_col7","_col9","_col3"] - Merge Join Operator [MERGEJOIN_99] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_92] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_98] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_91] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_97] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_89] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 7 PREHOOK: query: with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/perf/query40.q.out ql/src/test/results/clientpositive/perf/query40.q.out index a45fdaf..1caa2c8 100644 --- ql/src/test/results/clientpositive/perf/query40.q.out +++ ql/src/test/results/clientpositive/perf/query40.q.out @@ -55,94 +55,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_34] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Group By Operator [GBY_31] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1 - Group By Operator [GBY_29] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1 - Select Operator [SEL_27] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_58] (rows=421645953 width=135) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col7","_col9","_col11","_col14"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_state"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_57] (rows=383314495 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col4","_col7","_col9","_col11"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=51333 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_53] (rows=51333 width=1436) - predicate:(i_current_price BETWEEN 0.99 AND 1.49 and i_item_sk is not null) - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_current_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_56] (rows=348467716 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2","_col4","_col7","_col9"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_52] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-03-08 23:00:00.0 AND 1998-05-08 00:00:00.0 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=316788826 width=135) - Conds:RS_15._col2, _col3=RS_16._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2, _col3 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_50] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_warehouse_sk","cs_item_sk","cs_order_number","cs_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_51] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_3] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query42.q.out ql/src/test/results/clientpositive/perf/query42.q.out index 7100bb3..58e1465 100644 --- ql/src/test/results/clientpositive/perf/query42.q.out +++ ql/src/test/results/clientpositive/perf/query42.q.out @@ -5,66 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_23] - Limit [LIM_22] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_33] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category_id","i_category","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_32] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_30] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query43.q.out ql/src/test/results/clientpositive/perf/query43.q.out index 72712e8..846ab70 100644 --- ql/src/test/results/clientpositive/perf/query43.q.out +++ ql/src/test/results/clientpositive/perf/query43.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col5","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=852 width=1910) - predicate:((s_gmt_offset = -6) and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name","s_gmt_offset"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_31] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_day_name"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query45.q.out ql/src/test/results/clientpositive/perf/query45.q.out index e2d0da5..0c601f8 100644 --- ql/src/test/results/clientpositive/perf/query45.q.out +++ ql/src/test/results/clientpositive/perf/query45.q.out @@ -5,117 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 9 <- Reducer 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_47] - Limit [LIM_46] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_45] (rows=95833781 width=135) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_44] - Group By Operator [GBY_42] (rows=95833781 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Group By Operator [GBY_40] (rows=191667562 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col11)"],keys:_col4, _col3 - Select Operator [SEL_39] (rows=191667562 width=135) - Output:["_col4","_col3","_col11"] - Merge Join Operator [MERGEJOIN_74] (rows=191667562 width=135) - Conds:RS_36._col0=RS_37._col5(Inner),Output:["_col3","_col4","_col11"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_70] (rows=88000001 width=860) - Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_64] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_65] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_zip"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_73] (rows=174243235 width=135) - Conds:RS_29._col0=RS_30._col1(Inner),Output:["_col5","_col6"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_72] (rows=158402938 width=135) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_68] (rows=144002668 width=135) - predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null and ws_item_sk is not null) - TableScan [TS_16] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_69] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_19] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_71] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_66] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_14] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_11] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_67] (rows=231000 width=1436) - predicate:(i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) - TableScan [TS_9] (rows=462000 width=1436) - default@item,i2,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query46.q.out ql/src/test/results/clientpositive/perf/query46.q.out index d0275c6..78399d5 100644 --- ql/src/test/results/clientpositive/perf/query46.q.out +++ ql/src/test/results/clientpositive/perf/query46.q.out @@ -5,130 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 15 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 12 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_50] - Limit [LIM_49] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_48] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Select Operator [SEL_46] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_45] (rows=463823414 width=88) - predicate:(_col5 <> _col8) - Merge Join Operator [MERGEJOIN_86] (rows=463823414 width=88) - Conds:RS_42._col0=RS_43._col1(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9","_col10"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_37] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_36] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_34] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col6)","sum(_col7)"],keys:_col1, _col17, _col3, _col5 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col17","_col3","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_85] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col17"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=766650239 width=88) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=7200 width=107) - Output:["_col0"] - Filter Operator [FIL_79] (rows=7200 width=107) - predicate:(((hd_dep_count = 4) or (hd_vehicle_count = 2)) and hd_demo_sk is not null) - TableScan [TS_15] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=696954748 width=88) - Conds:RS_24._col4=RS_25._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_78] (rows=852 width=1910) - predicate:((s_city) IN ('Rosedale', 'Bethlehem', 'Clinton', 'Clifton', 'Springfield') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_city"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_82] (rows=633595212 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=18263 width=1119) - Output:["_col0"] - Filter Operator [FIL_77] (rows=18263 width=1119) - predicate:((d_dow) IN (6, 0) and (d_year) IN (1998, 1999, 2000) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_76] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_customer_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_coupon_amt","ss_net_profit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=88000001 width=860) - Conds:RS_39._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_74] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,current_addr,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query48.q.out ql/src/test/results/clientpositive/perf/query48.q.out index 3b390f2..1eea1e7 100644 --- ql/src/test/results/clientpositive/perf/query48.q.out +++ ql/src/test/results/clientpositive/perf/query48.q.out @@ -5,90 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 6 <- Map 9 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_35] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_32] - Group By Operator [GBY_31] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(_col6)"] - Merge Join Operator [MERGEJOIN_58] (rows=15616946 width=88) - Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_50] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=14197224 width=88) - Output:["_col0","_col4"] - Filter Operator [FIL_25] (rows=14197224 width=88) - predicate:(((_col12) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 0 AND 2000) or ((_col12) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 3000) or ((_col12) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 25000)) - Select Operator [SEL_24] (rows=85183356 width=88) - Output:["_col0","_col4","_col6","_col12"] - Merge Join Operator [MERGEJOIN_57] (rows=85183356 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col5","_col7","_col9"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=465450 width=385) - Output:["_col0"] - Filter Operator [FIL_54] (rows=465450 width=385) - predicate:((cd_marital_status = 'M') and (cd_education_status = '4 yr Degree') and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_56] (rows=77439413 width=88) - Conds:RS_18._col3=RS_19._col0(Inner),Output:["_col1","_col2","_col5","_col7","_col9"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_53] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_9] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_55] (rows=70399465 width=88) - Conds:RS_15._col0=RS_16._col3(Inner),Output:["_col1","_col2","_col3","_col5","_col7"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_51] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_3] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col3 - Select Operator [SEL_8] (rows=63999513 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] - Filter Operator [FIL_52] (rows=63999513 width=88) - predicate:((ss_sales_price BETWEEN 100 AND 150 or ss_sales_price BETWEEN 50 AND 100 or ss_sales_price BETWEEN 150 AND 200) and (ss_net_profit BETWEEN 0 AND 2000 or ss_net_profit BETWEEN 150 AND 3000 or ss_net_profit BETWEEN 50 AND 25000) and ss_store_sk is not null and ss_cdemo_sk is not null and ss_addr_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_cdemo_sk","ss_addr_sk","ss_store_sk","ss_quantity","ss_sales_price","ss_net_profit"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query5.q.out ql/src/test/results/clientpositive/perf/query5.q.out index d3ec922..439dd6d 100644 --- ql/src/test/results/clientpositive/perf/query5.q.out +++ ql/src/test/results/clientpositive/perf/query5.q.out @@ -255,235 +255,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 17 <- Union 13 (CONTAINS) -Map 20 <- Union 21 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Union 13 (SIMPLE_EDGE) -Reducer 15 <- Map 19 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 22 <- Map 28 (SIMPLE_EDGE), Union 21 (SIMPLE_EDGE) -Reducer 23 <- Map 29 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE), Union 21 (CONTAINS) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 6 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Union 13 (CONTAINS) +Vertex 20 <- Union 21 (CONTAINS) +Vertex 22 <- Union 21 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 24 <- Union 6 (CONTAINS), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Union 21 (CONTAINS), Vertex 25 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_93] - Limit [LIM_92] (rows=100 width=110) - Number of rows:100 - Select Operator [SEL_91] (rows=1136898901 width=110) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_90] - Select Operator [SEL_89] (rows=1136898901 width=110) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_88] (rows=1136898901 width=110) - Output:["_col0","_col1","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 16 [CONTAINS] - Reduce Output Operator [RS_87] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_86] (rows=2273797803 width=110) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_49] (rows=191657181 width=132) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_48] (rows=191657181 width=132) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_46] (rows=383314363 width=132) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col4)","sum(_col3)","sum(_col5)"],keys:_col9 - Select Operator [SEL_45] (rows=383314363 width=132) - Output:["_col9","_col2","_col4","_col3","_col5"] - Merge Join Operator [MERGEJOIN_136] (rows=383314363 width=132) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=46000 width=460) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=46000 width=460) - predicate:cp_catalog_page_sk is not null - TableScan [TS_36] (rows=46000 width=460) - default@catalog_page,catalog_page,Tbl:COMPLETE,Col:NONE,Output:["cp_catalog_page_sk","cp_catalog_page_id"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_135] (rows=348467596 width=132) - Conds:Union 13._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_125] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - TableScan [TS_33] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_27] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_catalog_page_sk is not null) - TableScan [TS_25] (rows=287989836 width=135) - Output:["cs_sold_date_sk","cs_catalog_page_sk","cs_ext_sales_price","cs_net_profit"] - <-Map 17 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_30] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_124] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_catalog_page_sk is not null) - TableScan [TS_28] (rows=28798881 width=106) - Output:["cr_returned_date_sk","cr_catalog_page_sk","cr_return_amount","cr_net_loss"] - <-Reducer 24 [CONTAINS] - Reduce Output Operator [RS_87] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_86] (rows=2273797803 width=110) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_83] (rows=182955399 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_82] (rows=182955399 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Group By Operator [GBY_80] (rows=365910798 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col4)","sum(_col3)","sum(_col5)"],keys:_col9 - Select Operator [SEL_79] (rows=365910798 width=135) - Output:["_col9","_col2","_col4","_col3","_col5"] - Merge Join Operator [MERGEJOIN_138] (rows=365910798 width=135) - Conds:RS_76._col0=RS_77._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=84 width=1850) - Output:["_col0","_col1"] - Filter Operator [FIL_131] (rows=84 width=1850) - predicate:web_site_sk is not null - TableScan [TS_70] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_site_id"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_137] (rows=332646173 width=135) - Conds:Union 21._col1=RS_74._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_130] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - TableScan [TS_67] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Union 21 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] - Reduce Output Operator [RS_73] - PartitionCols:_col1 - Select Operator [SEL_54] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_127] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_web_site_sk is not null) - TableScan [TS_52] (rows=144002668 width=135) - Output:["ws_sold_date_sk","ws_web_site_sk","ws_ext_sales_price","ws_net_profit"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_73] - PartitionCols:_col1 - Select Operator [SEL_64] (rows=158402938 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_132] (rows=158402938 width=135) - Conds:RS_61._col0, _col2=RS_62._col1, _col2(Inner),Output:["_col1","_col3","_col6","_col7"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0, _col2 - Select Operator [SEL_57] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_128] (rows=144002668 width=135) - predicate:(ws_web_site_sk is not null and ws_order_number is not null and ws_item_sk is not null) - TableScan [TS_55] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_item_sk","ws_web_site_sk","ws_order_number"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col1, _col2 - Select Operator [SEL_60] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_129] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_item_sk is not null and wr_order_number is not null) - TableScan [TS_58] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_order_number","wr_return_amt","wr_net_loss"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_87] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_86] (rows=2273797803 width=110) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_24] (rows=383320021 width=87) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_23] (rows=383320021 width=87) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Group By Operator [GBY_21] (rows=766640042 width=87) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col4)","sum(_col3)","sum(_col5)"],keys:_col9 - Select Operator [SEL_20] (rows=766640042 width=87) - Output:["_col9","_col2","_col4","_col3","_col5"] - Merge Join Operator [MERGEJOIN_134] (rows=766640042 width=87) - Conds:RS_17._col0=RS_18._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_122] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_11] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_133] (rows=696945478 width=87) - Conds:Union 2._col1=RS_15._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_121] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - TableScan [TS_8] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_14] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_119] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - Output:["ss_sold_date_sk","ss_store_sk","ss_ext_sales_price","ss_net_profit"] - <-Map 9 [CONTAINS] - Reduce Output Operator [RS_14] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_120] (rows=57591150 width=77) - predicate:(sr_returned_date_sk is not null and sr_store_sk is not null) - TableScan [TS_3] (rows=57591150 width=77) - Output:["sr_returned_date_sk","sr_store_sk","sr_return_amt","sr_net_loss"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query50.q.out ql/src/test/results/clientpositive/perf/query50.q.out index 47a00b0..3507155 100644 --- ql/src/test/results/clientpositive/perf/query50.q.out +++ ql/src/test/results/clientpositive/perf/query50.q.out @@ -117,94 +117,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Group By Operator [GBY_31] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Group By Operator [GBY_29] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - Merge Join Operator [MERGEJOIN_59] (rows=843315281 width=88) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col0","_col5","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1704 width=1910) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Filter Operator [FIL_55] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_id","s_street_number","s_street_name","s_street_type","s_suite_number","s_city","s_county","s_state","s_zip"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=766650239 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=73049 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_57] (rows=696954748 width=88) - Conds:RS_18._col5=RS_19._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_53] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_56] (rows=633595212 width=88) - Conds:RS_15._col1, _col2, _col4=RS_16._col1, _col2, _col3(Inner),Output:["_col0","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_51] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_52] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_3] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query51.q.out ql/src/test/results/clientpositive/perf/query51.q.out index 55e08d9..9695d99 100644 --- ql/src/test/results/clientpositive/perf/query51.q.out +++ ql/src/test/results/clientpositive/perf/query51.q.out @@ -87,109 +87,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_53] - Limit [LIM_52] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_51] (rows=116159124 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_50] - Select Operator [SEL_46] (rows=116159124 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_58] (rows=116159124 width=88) - predicate:(max_window_0 > max_window_1) - PTF Operator [PTF_45] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END ASC NULLS FIRST","partition by:":"CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END"}] - Select Operator [SEL_44] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END - Merge Join Operator [MERGEJOIN_67] (rows=348477374 width=88) - Conds:RS_40._col0, _col1=RS_41._col0, _col1(Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=79201469 width=135) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_36] (rows=79201469 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_32] (rows=79201469 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=158402938 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Select Operator [SEL_29] (rows=158402938 width=135) - Output:["_col1","_col4","_col2"] - Merge Join Operator [MERGEJOIN_66] (rows=158402938 width=135) - Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_61] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_20] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_16] (rows=316797606 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col1","_col4","_col2"] - Merge Join Operator [MERGEJOIN_65] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_59] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_60] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query52.q.out ql/src/test/results/clientpositive/perf/query52.q.out index c61ed30..54dd921 100644 --- ql/src/test/results/clientpositive/perf/query52.q.out +++ ql/src/test/results/clientpositive/perf/query52.q.out @@ -5,66 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query54.q.out ql/src/test/results/clientpositive/perf/query54.q.out index 0b59b1b..5794748 100644 --- ql/src/test/results/clientpositive/perf/query54.q.out +++ ql/src/test/results/clientpositive/perf/query54.q.out @@ -5,175 +5,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 12 <- Union 13 (CONTAINS) -Map 18 <- Union 13 (CONTAINS) -Reducer 10 <- Reducer 17 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 19 (SIMPLE_EDGE), Union 13 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 13 (CONTAINS) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_69] - Limit [LIM_68] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_67] (rows=174238687 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_66] - Select Operator [SEL_65] (rows=174238687 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_64] (rows=174238687 width=88) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Group By Operator [GBY_62] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_60] (rows=348477374 width=88) - Output:["_col0"] - Group By Operator [GBY_59] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col10 - Select Operator [SEL_56] (rows=696954748 width=88) - Output:["_col10","_col2"] - Merge Join Operator [MERGEJOIN_117] (rows=696954748 width=88) - Conds:RS_53._col1=RS_54._col5(Inner),Output:["_col2","_col10"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_116] (rows=316240138 width=135) - Conds:RS_46._col0=RS_47._col1(Inner),Output:["_col5"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Select Operator [SEL_42] (rows=287491029 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_41] (rows=287491029 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Group By Operator [GBY_39] (rows=574982058 width=135) - Output:["_col0","_col1"],keys:_col10, _col9 - Select Operator [SEL_38] (rows=574982058 width=135) - Output:["_col10","_col9"] - Merge Join Operator [MERGEJOIN_115] (rows=574982058 width=135) - Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col9","_col10"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_110] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_26] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_114] (rows=522710951 width=135) - Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col1"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=115500 width=1436) - Output:["_col0"] - Filter Operator [FIL_109] (rows=115500 width=1436) - predicate:((i_category = 'Jewelry') and (i_class = 'football') and i_item_sk is not null) - TableScan [TS_23] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_113] (rows=475191764 width=135) - Conds:Union 13._col0=RS_30._col0(Inner),Output:["_col1","_col2"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_108] (rows=18262 width=1119) - predicate:((d_moy = 3) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] - Reduce Output Operator [RS_29] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_106] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_12] (rows=287989836 width=135) - Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] - <-Map 18 [CONTAINS] - Reduce Output Operator [RS_29] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_107] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_15] (rows=144002668 width=135) - Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_112] (rows=44000000 width=1014) - Conds:RS_43._col1, _col2=RS_44._col0, _col1(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_105] (rows=1704 width=1910) - predicate:(s_county is not null and s_state is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_county","s_state"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1, _col2 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_104] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null and ca_state is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_111] (rows=633595212 width=88) - Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_102] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_103] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1203 AND 1205 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query55.q.out ql/src/test/results/clientpositive/perf/query55.q.out index fb3743d..dc78857 100644 --- ql/src/test/results/clientpositive/perf/query55.q.out +++ ql/src/test/results/clientpositive/perf/query55.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 36) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query56.q.out ql/src/test/results/clientpositive/perf/query56.q.out index 4fa28c2..0035d9f 100644 --- ql/src/test/results/clientpositive/perf/query56.q.out +++ ql/src/test/results/clientpositive/perf/query56.q.out @@ -133,289 +133,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query58.q.out ql/src/test/results/clientpositive/perf/query58.q.out index f7e6a08..e6820ca 100644 --- ql/src/test/results/clientpositive/perf/query58.q.out +++ ql/src/test/results/clientpositive/perf/query58.q.out @@ -95,270 +95,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Map 23 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 26 <- Map 32 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Map 33 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE) -Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 18 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 28 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_113] - Limit [LIM_112] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_111] (rows=14606 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_110] - Select Operator [SEL_109] (rows=14606 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_108] (rows=14606 width=135) - predicate:(_col1 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col3 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col5 BETWEEN (0.9 * _col1) AND (1.1 * _col1) and _col5 BETWEEN (0.9 * _col3) AND (1.1 * _col3)) - Merge Join Operator [MERGEJOIN_209] (rows=95833780 width=135) - Conds:RS_104._col0=RS_105._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_102] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_100] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_99] (rows=174243235 width=135) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_207] (rows=174243235 width=135) - Conds:RS_96._col4=RS_97._col0(Inner),Output:["_col5","_col7"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_195] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_87] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_206] (rows=158402938 width=135) - Conds:RS_93._col0=RS_94._col0(Inner),Output:["_col4","_col5"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_194] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_84] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_205] (rows=88388 width=1119) - Conds:RS_90._col1=RS_91._col0(Inner),Output:["_col0"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col1 - Select Operator [SEL_73] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_71] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_204] (rows=80353 width=1119) - Conds:RS_80._col1=RS_81._col1(Inner),Output:["_col0"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col1 - Select Operator [SEL_76] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_192] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_74] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col1 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_193] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Filter Operator [FIL_69] (rows=4732408 width=88) - predicate:(_col1 BETWEEN (0.9 * _col3) AND (1.1 * _col3) and _col3 BETWEEN (0.9 * _col1) AND (1.1 * _col1)) - Merge Join Operator [MERGEJOIN_208] (rows=383325119 width=88) - Conds:RS_66._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col0 - Group By Operator [GBY_64] (rows=174233858 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Group By Operator [GBY_62] (rows=348467716 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_61] (rows=348467716 width=135) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_203] (rows=348467716 width=135) - Conds:RS_58._col4=RS_59._col0(Inner),Output:["_col5","_col7"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_190] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_49] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_202] (rows=316788826 width=135) - Conds:RS_55._col0=RS_56._col0(Inner),Output:["_col4","_col5"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_189] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_46] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_201] (rows=88388 width=1119) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_186] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_33] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_200] (rows=80353 width=1119) - Conds:RS_42._col1=RS_43._col1(Inner),Output:["_col0"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_187] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_188] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_28] (rows=696954748 width=88) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_199] (rows=696954748 width=88) - Conds:RS_25._col4=RS_26._col0(Inner),Output:["_col5","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_185] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_16] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_198] (rows=633595212 width=88) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col4","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_184] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_13] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_197] (rows=88388 width=1119) - Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_181] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_196] (rows=80353 width=1119) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_183] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_182] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query6.q.out ql/src/test/results/clientpositive/perf/query6.q.out index 094459e..79b6220 100644 --- ql/src/test/results/clientpositive/perf/query6.q.out +++ ql/src/test/results/clientpositive/perf/query6.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join MERGEJOIN[196][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[193][tables = [$hdt$_5, $hdt$_6]] in Stage 'Reducer 21' is a cross product -Warning: Shuffle Join MERGEJOIN[194][tables = [$hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 22' is a cross product -Warning: Shuffle Join MERGEJOIN[195][tables = [$hdt$_5, $hdt$_6, $hdt$_7, $hdt$_8, $hdt$_9, $hdt$_10, $hdt$_11]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[196][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[193][tables = [$hdt$_5, $hdt$_6]] in Stage 'Vertex 21' is a cross product +Warning: Shuffle Join MERGEJOIN[194][tables = [$hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Vertex 22' is a cross product +Warning: Shuffle Join MERGEJOIN[195][tables = [$hdt$_5, $hdt$_6, $hdt$_7, $hdt$_8, $hdt$_9, $hdt$_10, $hdt$_11]] in Stage 'Vertex 23' is a cross product PREHOOK: query: explain select a.ca_state state, count(*) cnt from customer_address a ,customer c @@ -53,280 +53,31 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) -Reducer 12 <- Map 11 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 18 <- Map 17 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 10 (CUSTOM_SIMPLE_EDGE) -Reducer 21 <- Map 20 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (CUSTOM_SIMPLE_EDGE), Reducer 29 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Map 30 (CUSTOM_SIMPLE_EDGE), Map 31 (CUSTOM_SIMPLE_EDGE), Map 32 (CUSTOM_SIMPLE_EDGE), Map 33 (CUSTOM_SIMPLE_EDGE), Reducer 22 (CUSTOM_SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (CUSTOM_SIMPLE_EDGE) -Reducer 29 <- Map 28 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 36 (SIMPLE_EDGE) -Reducer 4 <- Reducer 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 35 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 10 (PARTITION_EDGE) +Vertex 21 <- Vertex 20 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 29 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE), Vertex 31 (PARTITION_EDGE), Vertex 32 (PARTITION_EDGE), Vertex 33 (PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_129] - Limit [LIM_128] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_127] (rows=187554724582209678 width=1) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_126] - Filter Operator [FIL_124] (rows=187554724582209678 width=1) - predicate:(_col1 >= 10) - Group By Operator [GBY_123] (rows=562664173746629034 width=1) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Group By Operator [GBY_121] (rows=1125328347493258069 width=1) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col1 - Select Operator [SEL_120] (rows=1125328347493258069 width=1) - Output:["_col1"] - Filter Operator [FIL_119] (rows=1125328347493258069 width=1) - predicate:(_col10 > (1.2 * CASE WHEN (_col15 is null) THEN (null) ELSE (_col14) END)) - Select Operator [SEL_118] (rows=3375985042479774208 width=1) - Output:["_col1","_col10","_col14","_col15"] - Merge Join Operator [MERGEJOIN_203] (rows=3375985042479774208 width=1) - Conds:RS_115._col2=RS_116._col0(Inner),Output:["_col8","_col10","_col11","_col16"] - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_200] (rows=88000001 width=860) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col3"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_98] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_customer_sk is not null) - TableScan [TS_96] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_192] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_99] (rows=40000000 width=1014) - default@customer_address,a,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_202] (rows=3069077244824791552 width=1) - Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col2","_col8","_col10","_col11"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_199] (rows=2790070162094850048 width=1) - Conds:RS_92._col2=RS_93._col2(Left Outer),Output:["_col0","_col1","_col3","_col4"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col2 - Select Operator [SEL_33] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_186] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_31] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_category"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col2 - Select Operator [SEL_91] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_90] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Group By Operator [GBY_88] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],aggregations:["avg(_col0)"],keys:_col2 - Select Operator [SEL_87] (rows=5072854730221289472 width=1) - Output:["_col2","_col0"] - Merge Join Operator [MERGEJOIN_198] (rows=5072854730221289472 width=1) - Conds:RS_84._col1=RS_85._col0(Inner),Output:["_col0","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=462000 width=1436) - Output:["_col0","_col1"] - TableScan [TS_34] (rows=462000 width=1436) - default@item,j,Tbl:COMPLETE,Col:NONE,Output:["i_current_price","i_category"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_85] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Group By Operator [GBY_80] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_195] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),(Inner),Output:["_col3"] - <-Map 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_74] - Select Operator [SEL_60] (rows=462000 width=1436) - Output:["_col0"] - TableScan [TS_59] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_category"] - <-Map 31 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_75] - Select Operator [SEL_62] (rows=40000000 width=4) - TableScan [TS_61] (rows=40000000 width=1014) - default@customer_address,a,Tbl:COMPLETE,Col:COMPLETE - <-Map 32 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_76] - Select Operator [SEL_64] (rows=80000000 width=4) - TableScan [TS_63] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:COMPLETE - <-Map 33 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_77] - Select Operator [SEL_66] (rows=575995635 width=4) - TableScan [TS_65] (rows=575995635 width=88) - default@store_sales,s,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_73] - Merge Join Operator [MERGEJOIN_194] (rows=667010419 width=1133) - Conds:(Left Outer) - <-Reducer 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_70] - Merge Join Operator [MERGEJOIN_193] (rows=73049 width=13) - Conds:(Inner) - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_67] - Select Operator [SEL_37] (rows=73049 width=4) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_68] - Select Operator [SEL_51] (rows=1 width=8) - Filter Operator [FIL_50] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 26 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_47] - Group By Operator [GBY_46] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_44] (rows=9131 width=1119) - Group By Operator [GBY_43] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_40] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_188] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2)) - TableScan [TS_38] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 29 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_58] (rows=9131 width=1119) - Group By Operator [GBY_57] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Group By Operator [GBY_55] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_54] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_189] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2)) - TableScan [TS_52] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_201] (rows=633595212 width=97) - Conds:RS_109._col0=RS_110._col0(Inner),Output:["_col1","_col2"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_197] (rows=80353 width=1119) - Conds:RS_27._col1=RS_28._col0(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_184] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_month_seq is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_22] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_185] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2) and d_month_seq is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_196] (rows=575995635 width=97) - Conds:(Inner),Output:["_col0","_col1","_col2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_106] - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_182] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,s,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Reducer 10 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_107] - Select Operator [SEL_16] (rows=1 width=8) - Filter Operator [FIL_15] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_13] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 9 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_12] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_9] (rows=9131 width=1119) - Group By Operator [GBY_8] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_183] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2)) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query60.q.out ql/src/test/results/clientpositive/perf/query60.q.out index ad9d08e..f050ef6 100644 --- ql/src/test/results/clientpositive/perf/query60.q.out +++ ql/src/test/results/clientpositive/perf/query60.q.out @@ -153,289 +153,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query64.q.out ql/src/test/results/clientpositive/perf/query64.q.out index 468c1bc..115e5e6 100644 --- ql/src/test/results/clientpositive/perf/query64.q.out +++ ql/src/test/results/clientpositive/perf/query64.q.out @@ -5,657 +5,52 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 12 <- Map 40 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Map 41 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 17 <- Map 23 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 24 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 34 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 35 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 30 (SIMPLE_EDGE) -Reducer 27 <- Map 31 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 37 <- Map 36 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 44 <- Reducer 43 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 45 <- Reducer 44 (SIMPLE_EDGE) -Reducer 48 <- Map 47 (SIMPLE_EDGE), Map 53 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE), Reducer 60 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 45 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (SIMPLE_EDGE), Reducer 77 (SIMPLE_EDGE) -Reducer 51 <- Map 79 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE) -Reducer 52 <- Map 80 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) -Reducer 55 <- Map 54 (SIMPLE_EDGE), Map 61 (SIMPLE_EDGE) -Reducer 56 <- Map 62 (SIMPLE_EDGE), Reducer 55 (SIMPLE_EDGE) -Reducer 57 <- Map 63 (SIMPLE_EDGE), Reducer 56 (SIMPLE_EDGE) -Reducer 58 <- Reducer 57 (SIMPLE_EDGE), Reducer 68 (SIMPLE_EDGE) -Reducer 59 <- Map 73 (SIMPLE_EDGE), Reducer 58 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 60 <- Map 74 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 65 <- Map 64 (SIMPLE_EDGE), Map 69 (SIMPLE_EDGE) -Reducer 66 <- Map 70 (SIMPLE_EDGE), Reducer 65 (SIMPLE_EDGE) -Reducer 67 <- Map 71 (SIMPLE_EDGE), Reducer 66 (SIMPLE_EDGE) -Reducer 68 <- Map 72 (SIMPLE_EDGE), Reducer 67 (SIMPLE_EDGE) -Reducer 76 <- Map 75 (SIMPLE_EDGE), Map 78 (SIMPLE_EDGE) -Reducer 77 <- Reducer 76 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 52 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (SORT_PARTITION_EDGE) +Vertex 48 <- Vertex 47 (SORT_PARTITION_EDGE), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (SORT_PARTITION_EDGE), Vertex 60 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 45 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (SORT_PARTITION_EDGE), Vertex 77 (SORT_PARTITION_EDGE) +Vertex 51 <- Vertex 50 (SORT_PARTITION_EDGE), Vertex 79 (SORT_PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 80 (SORT_PARTITION_EDGE) +Vertex 55 <- Vertex 54 (SORT_PARTITION_EDGE), Vertex 61 (SORT_PARTITION_EDGE) +Vertex 56 <- Vertex 55 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 57 <- Vertex 56 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 58 <- Vertex 57 (SORT_PARTITION_EDGE), Vertex 68 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 73 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 74 (SORT_PARTITION_EDGE) +Vertex 65 <- Vertex 64 (SORT_PARTITION_EDGE), Vertex 69 (SORT_PARTITION_EDGE) +Vertex 66 <- Vertex 65 (SORT_PARTITION_EDGE), Vertex 70 (SORT_PARTITION_EDGE) +Vertex 67 <- Vertex 66 (SORT_PARTITION_EDGE), Vertex 71 (SORT_PARTITION_EDGE) +Vertex 68 <- Vertex 67 (SORT_PARTITION_EDGE), Vertex 72 (SORT_PARTITION_EDGE) +Vertex 76 <- Vertex 75 (SORT_PARTITION_EDGE), Vertex 78 (SORT_PARTITION_EDGE) +Vertex 77 <- Vertex 76 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_267] - Select Operator [SEL_266] (rows=331415616 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_265] - Select Operator [SEL_264] (rows=331415616 width=88) - Output:["_col0","_col1","_col10","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col2","_col20","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_263] (rows=331415616 width=88) - predicate:(_col30 <= _col13) - Merge Join Operator [MERGEJOIN_658] (rows=994246850 width=88) - Conds:RS_260._col2, _col1, _col3=RS_261._col2, _col1, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_260] - PartitionCols:_col2, _col1, _col3 - Select Operator [SEL_128] (rows=903860754 width=88) - Output:["_col0","_col1","_col10","_col11","_col13","_col14","_col15","_col16","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_127] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Group By Operator [GBY_125] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col44, _col50, _col45, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col53 - Select Operator [SEL_124] (rows=1807721509 width=88) - Output:["_col44","_col50","_col45","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col53","_col26","_col27","_col28"] - Merge Join Operator [MERGEJOIN_656] (rows=1807721509 width=88) - Conds:RS_121._col0=RS_122._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col18 - Select Operator [SEL_117] (rows=1643383155 width=88) - Output:["_col1","_col18","_col2","_col23","_col24","_col25","_col3","_col37","_col39","_col4","_col41","_col42","_col47","_col50","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_638] (rows=1643383155 width=88) - Conds:RS_114._col21=RS_115._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48","_col50","_col51","_col52","_col53"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_602] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_99] (rows=40000000 width=1014) - default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col21 - Merge Join Operator [MERGEJOIN_637] (rows=1493984654 width=88) - Conds:RS_111._col9=RS_112._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_601] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_96] (rows=40000000 width=1014) - default@customer_address,ad1,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_636] (rows=1358167838 width=88) - Conds:RS_108._col5=RS_109._col0(Inner),Output:["_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_635] (rows=1234698008 width=88) - Conds:RS_105._col0=RS_106._col17(Inner),Output:["_col5","_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col17 - Select Operator [SEL_79] (rows=1122452711 width=88) - Output:["_col10","_col11","_col12","_col17","_col18","_col2","_col24","_col26","_col28","_col29","_col34","_col37","_col5","_col6"] - Merge Join Operator [MERGEJOIN_633] (rows=1122452711 width=88) - Conds:RS_76._col7=RS_77._col0(Inner),Output:["_col1","_col4","_col5","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_598] (rows=2300 width=1179) - predicate:p_promo_sk is not null - TableScan [TS_70] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_632] (rows=1020411534 width=88) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_597] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - TableScan [TS_67] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Select Operator [SEL_66] (rows=927646829 width=88) - Output:["_col1","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col4","_col5","_col7","_col9"] - Filter Operator [FIL_65] (rows=927646829 width=88) - predicate:(_col30 <> _col32) - Select Operator [SEL_64] (rows=927646829 width=88) - Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col30","_col32"] - Merge Join Operator [MERGEJOIN_631] (rows=927646829 width=88) - Conds:RS_61._col0=RS_62._col4(Inner),Output:["_col2","_col3","_col7","_col9","_col11","_col13","_col15","_col18","_col19","_col21","_col23","_col24","_col25","_col31","_col32"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_626] (rows=106480005 width=860) - Conds:RS_58._col1=RS_59._col0(Inner),Output:["_col0","_col2","_col3","_col7","_col9","_col11"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_591] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_21] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_625] (rows=96800003 width=860) - Conds:RS_55._col4=RS_56._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_590] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_18] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_624] (rows=88000001 width=860) - Conds:RS_52._col5=RS_53._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col5 - Select Operator [SEL_14] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_588] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - TableScan [TS_12] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_589] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col4 - Select Operator [SEL_51] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col13","_col19","_col20","_col3","_col4","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_630] (rows=843315281 width=88) - Conds:RS_48._col3=RS_49._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18","_col20"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_596] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_36] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_629] (rows=766650239 width=88) - Conds:RS_45._col6=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_595] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - TableScan [TS_33] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_628] (rows=696954748 width=88) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_594] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_30] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_627] (rows=633595212 width=88) - Conds:RS_39._col1, _col8=RS_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1, _col8 - Select Operator [SEL_26] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_592] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_customer_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - TableScan [TS_24] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Select Operator [SEL_29] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_593] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_27] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_623] (rows=7920 width=107) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_587] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_9] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_586] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_6] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_95] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_94] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_93] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Group By Operator [GBY_91] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_89] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_634] (rows=316788826 width=135) - Conds:RS_86._col0, _col1=RS_87._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0, _col1 - Select Operator [SEL_82] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_599] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - TableScan [TS_80] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_600] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_622] (rows=7920 width=107) - Conds:RS_118._col1=RS_119._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_584] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_0] (rows=7200 width=107) - default@household_demographics,hd1,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_585] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_3] (rows=20 width=12) - default@income_band,ib1,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_261] - PartitionCols:_col2, _col1, _col3 - Select Operator [SEL_258] (rows=903860754 width=88) - Output:["_col1","_col13","_col14","_col15","_col16","_col2","_col3"] - Group By Operator [GBY_257] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_256] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Group By Operator [GBY_255] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col44, _col50, _col45, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col53 - Select Operator [SEL_254] (rows=1807721509 width=88) - Output:["_col44","_col50","_col45","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col53","_col26","_col27","_col28"] - Merge Join Operator [MERGEJOIN_657] (rows=1807721509 width=88) - Conds:RS_251._col0=RS_252._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_251] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_639] (rows=7920 width=107) - Conds:RS_248._col1=RS_249._col0(Inner),Output:["_col0"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_248] - PartitionCols:_col1 - Select Operator [SEL_132] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_603] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_130] (rows=7200 width=107) - default@household_demographics,hd1,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_249] - PartitionCols:_col0 - Select Operator [SEL_135] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_604] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_133] (rows=20 width=12) - default@income_band,ib1,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_252] - PartitionCols:_col18 - Select Operator [SEL_247] (rows=1643383155 width=88) - Output:["_col1","_col18","_col2","_col23","_col24","_col25","_col3","_col37","_col39","_col4","_col41","_col42","_col47","_col50","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_655] (rows=1643383155 width=88) - Conds:RS_244._col21=RS_245._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48","_col50","_col51","_col52","_col53"] - <-Map 80 [SIMPLE_EDGE] - SHUFFLE [RS_245] - PartitionCols:_col0 - Select Operator [SEL_231] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_621] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_229] (rows=40000000 width=1014) - default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_244] - PartitionCols:_col21 - Merge Join Operator [MERGEJOIN_654] (rows=1493984654 width=88) - Conds:RS_241._col9=RS_242._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48"] - <-Map 79 [SIMPLE_EDGE] - SHUFFLE [RS_242] - PartitionCols:_col0 - Select Operator [SEL_228] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_620] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_226] (rows=40000000 width=1014) - default@customer_address,ad1,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_241] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_653] (rows=1358167838 width=88) - Conds:RS_238._col5=RS_239._col0(Inner),Output:["_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 49 [SIMPLE_EDGE] - SHUFFLE [RS_238] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_652] (rows=1234698008 width=88) - Conds:RS_235._col0=RS_236._col17(Inner),Output:["_col5","_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_235] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_640] (rows=7920 width=107) - Conds:RS_232._col1=RS_233._col0(Inner),Output:["_col0"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_232] - PartitionCols:_col1 - Select Operator [SEL_138] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_605] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_136] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 53 [SIMPLE_EDGE] - SHUFFLE [RS_233] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_606] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_139] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_236] - PartitionCols:_col17 - Select Operator [SEL_209] (rows=1122452711 width=88) - Output:["_col10","_col11","_col12","_col17","_col18","_col2","_col24","_col26","_col28","_col29","_col34","_col37","_col5","_col6"] - Merge Join Operator [MERGEJOIN_650] (rows=1122452711 width=88) - Conds:RS_206._col7=RS_207._col0(Inner),Output:["_col1","_col4","_col5","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 74 [SIMPLE_EDGE] - SHUFFLE [RS_207] - PartitionCols:_col0 - Select Operator [SEL_202] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_617] (rows=2300 width=1179) - predicate:p_promo_sk is not null - TableScan [TS_200] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_206] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_649] (rows=1020411534 width=88) - Conds:RS_203._col1=RS_204._col0(Inner),Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 73 [SIMPLE_EDGE] - SHUFFLE [RS_204] - PartitionCols:_col0 - Select Operator [SEL_199] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_616] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - TableScan [TS_197] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 58 [SIMPLE_EDGE] - SHUFFLE [RS_203] - PartitionCols:_col1 - Select Operator [SEL_196] (rows=927646829 width=88) - Output:["_col1","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col4","_col5","_col7","_col9"] - Filter Operator [FIL_195] (rows=927646829 width=88) - predicate:(_col30 <> _col32) - Select Operator [SEL_194] (rows=927646829 width=88) - Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col30","_col32"] - Merge Join Operator [MERGEJOIN_648] (rows=927646829 width=88) - Conds:RS_191._col0=RS_192._col4(Inner),Output:["_col2","_col3","_col7","_col9","_col11","_col13","_col15","_col18","_col19","_col21","_col23","_col24","_col25","_col31","_col32"] - <-Reducer 57 [SIMPLE_EDGE] - SHUFFLE [RS_191] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_643] (rows=106480005 width=860) - Conds:RS_188._col1=RS_189._col0(Inner),Output:["_col0","_col2","_col3","_col7","_col9","_col11"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_189] - PartitionCols:_col0 - Select Operator [SEL_153] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_610] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_151] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 56 [SIMPLE_EDGE] - SHUFFLE [RS_188] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_642] (rows=96800003 width=860) - Conds:RS_185._col4=RS_186._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 62 [SIMPLE_EDGE] - SHUFFLE [RS_186] - PartitionCols:_col0 - Select Operator [SEL_150] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_609] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_148] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 55 [SIMPLE_EDGE] - SHUFFLE [RS_185] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_641] (rows=88000001 width=860) - Conds:RS_182._col5=RS_183._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 54 [SIMPLE_EDGE] - SHUFFLE [RS_182] - PartitionCols:_col5 - Select Operator [SEL_144] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_607] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - TableScan [TS_142] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Map 61 [SIMPLE_EDGE] - SHUFFLE [RS_183] - PartitionCols:_col0 - Select Operator [SEL_147] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_608] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_145] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 68 [SIMPLE_EDGE] - SHUFFLE [RS_192] - PartitionCols:_col4 - Select Operator [SEL_181] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col13","_col19","_col20","_col3","_col4","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_647] (rows=843315281 width=88) - Conds:RS_178._col3=RS_179._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18","_col20"] - <-Map 72 [SIMPLE_EDGE] - SHUFFLE [RS_179] - PartitionCols:_col0 - Select Operator [SEL_168] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_615] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_166] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 67 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_646] (rows=766650239 width=88) - Conds:RS_175._col6=RS_176._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18"] - <-Map 71 [SIMPLE_EDGE] - SHUFFLE [RS_176] - PartitionCols:_col0 - Select Operator [SEL_165] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_614] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - TableScan [TS_163] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 66 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_645] (rows=696954748 width=88) - Conds:RS_172._col0=RS_173._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 70 [SIMPLE_EDGE] - SHUFFLE [RS_173] - PartitionCols:_col0 - Select Operator [SEL_162] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_613] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_160] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_644] (rows=633595212 width=88) - Conds:RS_169._col1, _col8=RS_170._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col1, _col8 - Select Operator [SEL_156] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_611] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_customer_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - TableScan [TS_154] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 69 [SIMPLE_EDGE] - SHUFFLE [RS_170] - PartitionCols:_col0, _col1 - Select Operator [SEL_159] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_612] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_157] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 77 [SIMPLE_EDGE] - SHUFFLE [RS_239] - PartitionCols:_col0 - Select Operator [SEL_225] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_224] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_223] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 76 [SIMPLE_EDGE] - SHUFFLE [RS_222] - PartitionCols:_col0 - Group By Operator [GBY_221] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_219] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_651] (rows=316788826 width=135) - Conds:RS_216._col0, _col1=RS_217._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 75 [SIMPLE_EDGE] - SHUFFLE [RS_216] - PartitionCols:_col0, _col1 - Select Operator [SEL_212] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_618] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - TableScan [TS_210] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 78 [SIMPLE_EDGE] - SHUFFLE [RS_217] - PartitionCols:_col0, _col1 - Select Operator [SEL_215] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_619] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - TableScan [TS_213] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query65.q.out ql/src/test/results/clientpositive/perf/query65.q.out index 060cddf..af6d33f 100644 --- ql/src/test/results/clientpositive/perf/query65.q.out +++ ql/src/test/results/clientpositive/perf/query65.q.out @@ -77,125 +77,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_52] - Limit [LIM_51] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_50] (rows=255550079 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_49] - Select Operator [SEL_48] (rows=255550079 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_47] (rows=255550079 width=88) - predicate:(_col11 <= (0.1 * _col8)) - Select Operator [SEL_46] (rows=766650239 width=88) - Output:["_col1","_col3","_col4","_col5","_col6","_col8","_col11"] - Merge Join Operator [MERGEJOIN_82] (rows=766650239 width=88) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col4","_col6","_col8","_col9","_col10","_col11"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_78] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_36] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc","i_current_price","i_wholesale_cost","i_brand"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_81] (rows=696954748 width=88) - Conds:RS_39._col0=RS_40._col0(Inner),RS_39._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col4","_col6"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_33] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=158398803 width=88) - Output:["_col0","_col1"] - Group By Operator [GBY_31] (rows=158398803 width=88) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Select Operator [SEL_27] (rows=316797606 width=88) - Output:["_col1","_col2"] - Group By Operator [GBY_26] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_23] (rows=633595212 width=88) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_80] (rows=633595212 width=88) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_75] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_14] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_79] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_73] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_74] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query66.q.out ql/src/test/results/clientpositive/perf/query66.q.out index 5cf60f6..6fd32bb 100644 --- ql/src/test/results/clientpositive/perf/query66.q.out +++ ql/src/test/results/clientpositive/perf/query66.q.out @@ -439,190 +439,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Map 22 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 23 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 7 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 9 - File Output Operator [FS_76] - Limit [LIM_75] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_74] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_73] - Select Operator [SEL_72] (rows=158120068 width=135) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col2","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col3","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col4","_col40","_col41","_col42","_col43","_col5","_col8","_col9"] - Group By Operator [GBY_71] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)","sum(VALUE._col24)","sum(VALUE._col25)","sum(VALUE._col26)","sum(VALUE._col27)","sum(VALUE._col28)","sum(VALUE._col29)","sum(VALUE._col30)","sum(VALUE._col31)","sum(VALUE._col32)","sum(VALUE._col33)","sum(VALUE._col34)","sum(VALUE._col35)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] - Group By Operator [GBY_64] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_62] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_60] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Merge Join Operator [MERGEJOIN_122] (rows=421645953 width=135) - Conds:RS_57._col3=RS_58._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_114] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_45] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_121] (rows=383314495 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_113] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_42] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_120] (rows=348467716 width=135) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_112] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=316788826 width=135) - Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_110] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_sold_date_sk is not null and cs_sold_time_sk is not null and cs_ship_mode_sk is not null) - TableScan [TS_33] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_ship_mode_sk","cs_warehouse_sk","cs_quantity","cs_ext_sales_price","cs_net_paid_inc_ship_tax"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_111] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] - Group By Operator [GBY_31] (rows=105417161 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_29] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_27] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_109] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_117] (rows=191667562 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_108] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_9] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_116] (rows=174243235 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_107] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_115] (rows=158402938 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_105] (rows=144002668 width=135) - predicate:(ws_warehouse_sk is not null and ws_sold_date_sk is not null and ws_sold_time_sk is not null and ws_ship_mode_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_ship_mode_sk","ws_warehouse_sk","ws_quantity","ws_sales_price","ws_net_paid_inc_tax"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_106] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] + Vertex 9 diff --git ql/src/test/results/clientpositive/perf/query67.q.out ql/src/test/results/clientpositive/perf/query67.q.out index 1f38027..2831261 100644 --- ql/src/test/results/clientpositive/perf/query67.q.out +++ ql/src/test/results/clientpositive/perf/query67.q.out @@ -87,93 +87,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_35] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_30] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_47] (rows=1149975358 width=88) - predicate:(rank_window_0 <= 100) - PTF Operator [PTF_29] (rows=3449926075 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col16 DESC NULLS LAST","partition by:":"_col0"}] - Select Operator [SEL_28] (rows=3449926075 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col16"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=3449926075 width=88) - Output:["_col0","_col1","_col16","_col2","_col3","_col4","_col5","_col6","_col7"] - Group By Operator [GBY_25] (rows=3449926075 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Group By Operator [GBY_23] (rows=6899852151 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, 0 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_54] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col4","_col7","_col8","_col9","_col11","_col13","_col14","_col15","_col16"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_51] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_53] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3","_col4","_col7","_col8","_col9","_col11"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_50] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_52] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col7","_col8","_col9"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_48] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_quantity","ss_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col2","_col3","_col4"] - Filter Operator [FIL_49] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq","d_year","d_moy","d_qoy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query68.q.out ql/src/test/results/clientpositive/perf/query68.q.out index b84bdf6..fb1b458 100644 --- ql/src/test/results/clientpositive/perf/query68.q.out +++ ql/src/test/results/clientpositive/perf/query68.q.out @@ -5,130 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 15 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 12 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_50] - Limit [LIM_49] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_48] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Select Operator [SEL_46] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_45] (rows=463823414 width=88) - predicate:(_col5 <> _col8) - Merge Join Operator [MERGEJOIN_86] (rows=463823414 width=88) - Conds:RS_42._col0=RS_43._col1(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9","_col10","_col11"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_37] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_36] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_34] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col1, _col18, _col3, _col5 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col18","_col3","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_85] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=766650239 width=88) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=7200 width=107) - Output:["_col0"] - Filter Operator [FIL_79] (rows=7200 width=107) - predicate:(((hd_dep_count = 4) or (hd_vehicle_count = 2)) and hd_demo_sk is not null) - TableScan [TS_15] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=696954748 width=88) - Conds:RS_24._col4=RS_25._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_78] (rows=852 width=1910) - predicate:((s_city) IN ('Rosedale', 'Bethlehem') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_city"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_82] (rows=633595212 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_77] (rows=4058 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Filter Operator [FIL_76] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_customer_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_ext_sales_price","ss_ext_list_price","ss_ext_tax"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=88000001 width=860) - Conds:RS_39._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_74] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,current_addr,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query69.q.out ql/src/test/results/clientpositive/perf/query69.q.out index 71ddd71..0bd5290 100644 --- ql/src/test/results/clientpositive/perf/query69.q.out +++ ql/src/test/results/clientpositive/perf/query69.q.out @@ -1,12 +1,12 @@ -Warning: Shuffle Join MERGEJOIN[261][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 17' is a cross product -Warning: Shuffle Join MERGEJOIN[262][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 22' is a cross product -Warning: Shuffle Join MERGEJOIN[273][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Reducer 13' is a cross product -Warning: Shuffle Join MERGEJOIN[263][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 33' is a cross product -Warning: Shuffle Join MERGEJOIN[264][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 38' is a cross product -Warning: Shuffle Join MERGEJOIN[275][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Reducer 29' is a cross product -Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 49' is a cross product -Warning: Shuffle Join MERGEJOIN[266][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 54' is a cross product -Warning: Shuffle Join MERGEJOIN[278][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Reducer 45' is a cross product +Warning: Shuffle Join MERGEJOIN[261][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 17' is a cross product +Warning: Shuffle Join MERGEJOIN[262][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 22' is a cross product +Warning: Shuffle Join MERGEJOIN[273][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Vertex 13' is a cross product +Warning: Shuffle Join MERGEJOIN[263][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 33' is a cross product +Warning: Shuffle Join MERGEJOIN[264][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 38' is a cross product +Warning: Shuffle Join MERGEJOIN[275][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Vertex 29' is a cross product +Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 49' is a cross product +Warning: Shuffle Join MERGEJOIN[266][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 54' is a cross product +Warning: Shuffle Join MERGEJOIN[278][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Vertex 45' is a cross product PREHOOK: query: explain select cd_gender, cd_marital_status, @@ -100,396 +100,40 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 23 (CUSTOM_SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE), Map 24 (CUSTOM_SIMPLE_EDGE), Map 25 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 39 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (CUSTOM_SIMPLE_EDGE), Map 35 (CUSTOM_SIMPLE_EDGE), Map 36 (CUSTOM_SIMPLE_EDGE) -Reducer 34 <- Reducer 33 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (CUSTOM_SIMPLE_EDGE), Map 40 (CUSTOM_SIMPLE_EDGE), Map 41 (CUSTOM_SIMPLE_EDGE) -Reducer 39 <- Reducer 38 (SIMPLE_EDGE) -Reducer 4 <- Reducer 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 47 (SIMPLE_EDGE) -Reducer 44 <- Reducer 43 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE) -Reducer 45 <- Reducer 44 (CUSTOM_SIMPLE_EDGE), Reducer 55 (CUSTOM_SIMPLE_EDGE) -Reducer 46 <- Reducer 45 (SIMPLE_EDGE) -Reducer 49 <- Map 48 (CUSTOM_SIMPLE_EDGE), Map 51 (CUSTOM_SIMPLE_EDGE), Map 52 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (SIMPLE_EDGE) -Reducer 54 <- Map 53 (CUSTOM_SIMPLE_EDGE), Map 56 (CUSTOM_SIMPLE_EDGE), Map 57 (CUSTOM_SIMPLE_EDGE) -Reducer 55 <- Reducer 54 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (PARTITION_EDGE), Vertex 23 (PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (PARTITION_EDGE), Vertex 19 (PARTITION_EDGE), Vertex 20 (PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 24 (PARTITION_EDGE), Vertex 25 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (PARTITION_EDGE), Vertex 39 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (PARTITION_EDGE), Vertex 35 (PARTITION_EDGE), Vertex 36 (PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (PARTITION_EDGE), Vertex 40 (PARTITION_EDGE), Vertex 41 (PARTITION_EDGE) +Vertex 39 <- Vertex 38 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 50 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (PARTITION_EDGE), Vertex 55 (PARTITION_EDGE) +Vertex 46 <- Vertex 45 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (PARTITION_EDGE), Vertex 51 (PARTITION_EDGE), Vertex 52 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (SORT_PARTITION_EDGE) +Vertex 54 <- Vertex 53 (PARTITION_EDGE), Vertex 56 (PARTITION_EDGE), Vertex 57 (PARTITION_EDGE) +Vertex 55 <- Vertex 54 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_184] - Limit [LIM_183] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_182] (rows=1268213682555322368 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_181] - Select Operator [SEL_180] (rows=1268213682555322368 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] - Group By Operator [GBY_179] (rows=1268213682555322368 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_177] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count()"],keys:_col6, _col7, _col8, _col9, _col10 - Select Operator [SEL_176] (rows=2536427365110644736 width=1) - Output:["_col6","_col7","_col8","_col9","_col10"] - Filter Operator [FIL_175] (rows=2536427365110644736 width=1) - predicate:_col15 is null - Merge Join Operator [MERGEJOIN_279] (rows=5072854730221289472 width=1) - Conds:RS_172._col0=RS_173._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col15"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Select Operator [SEL_121] (rows=4611686018427387903 width=1) - Output:["_col0","_col10","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_120] (rows=4611686018427387903 width=1) - predicate:_col13 is null - Select Operator [SEL_119] (rows=9223372036854775807 width=1) - Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col13"] - Merge Join Operator [MERGEJOIN_276] (rows=9223372036854775807 width=1) - Conds:RS_115._col0=RS_116._col0(Left Outer),RS_115._col0=RS_117._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_58] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_57] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Group By Operator [GBY_55] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_273] (rows=9223372036854775807 width=1) - Conds:(Inner),Output:["_col6"] - <-Reducer 12 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_51] - Merge Join Operator [MERGEJOIN_272] (rows=5072854730221289472 width=1) - Conds:RS_48._col1=RS_49._col0(Inner) - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_269] (rows=158402938 width=135) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_249] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_9] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_250] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_26] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_261] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 16 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_21] - Select Operator [SEL_16] (rows=40000000 width=4) - TableScan [TS_15] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_22] - Select Operator [SEL_18] (rows=1861800 width=4) - TableScan [TS_17] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Select Operator [SEL_20] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_19] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 23 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Group By Operator [GBY_43] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_262] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_36] - Select Operator [SEL_31] (rows=40000000 width=4) - TableScan [TS_30] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 24 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_37] - Select Operator [SEL_33] (rows=1861800 width=4) - TableScan [TS_32] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 25 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_38] - Select Operator [SEL_35] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_34] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_268] (rows=96800003 width=860) - Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1861800 width=385) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_248] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_6] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status","cd_purchase_estimate","cd_credit_rating"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_267] (rows=88000001 width=860) - Conds:RS_109._col2=RS_110._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_246] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_247] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Group By Operator [GBY_107] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col0 - Group By Operator [GBY_105] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_275] (rows=9223372036854775807 width=1) - Conds:(Inner),Output:["_col6"] - <-Reducer 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_101] - Merge Join Operator [MERGEJOIN_274] (rows=5072854730221289472 width=1) - Conds:RS_98._col1=RS_99._col0(Inner) - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_270] (rows=633595212 width=88) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_61] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_253] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_59] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_64] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_254] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_62] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Group By Operator [GBY_78] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Group By Operator [GBY_76] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_263] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 32 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_66] (rows=40000000 width=4) - TableScan [TS_65] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 35 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_72] - Select Operator [SEL_68] (rows=1861800 width=4) - TableScan [TS_67] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 36 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_73] - Select Operator [SEL_70] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_69] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 39 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_102] - Group By Operator [GBY_93] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Group By Operator [GBY_91] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_264] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 37 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_86] - Select Operator [SEL_81] (rows=40000000 width=4) - TableScan [TS_80] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 40 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_87] - Select Operator [SEL_83] (rows=1861800 width=4) - TableScan [TS_82] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 41 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_88] - Select Operator [SEL_85] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_84] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 46 [SIMPLE_EDGE] - SHUFFLE [RS_173] - PartitionCols:_col0 - Select Operator [SEL_171] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_170] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col0 - Group By Operator [GBY_168] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_278] (rows=9223372036854775807 width=1) - Conds:(Inner),Output:["_col6"] - <-Reducer 44 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_164] - Merge Join Operator [MERGEJOIN_277] (rows=5072854730221289472 width=1) - Conds:RS_161._col1=RS_162._col0(Inner) - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_161] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_271] (rows=316788826 width=135) - Conds:RS_158._col0=RS_159._col0(Inner),Output:["_col1"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_158] - PartitionCols:_col0 - Select Operator [SEL_124] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_257] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_122] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_customer_sk"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_159] - PartitionCols:_col0 - Select Operator [SEL_127] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_258] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_125] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_162] - PartitionCols:_col0 - Group By Operator [GBY_141] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 49 [SIMPLE_EDGE] - SHUFFLE [RS_140] - PartitionCols:_col0 - Group By Operator [GBY_139] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_265] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 48 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_134] - Select Operator [SEL_129] (rows=40000000 width=4) - TableScan [TS_128] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 51 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_135] - Select Operator [SEL_131] (rows=1861800 width=4) - TableScan [TS_130] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 52 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_136] - Select Operator [SEL_133] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_132] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 55 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_165] - Group By Operator [GBY_156] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 54 [SIMPLE_EDGE] - SHUFFLE [RS_155] - PartitionCols:_col0 - Group By Operator [GBY_154] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_266] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 53 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_149] - Select Operator [SEL_144] (rows=40000000 width=4) - TableScan [TS_143] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 56 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_150] - Select Operator [SEL_146] (rows=1861800 width=4) - TableScan [TS_145] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 57 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_151] - Select Operator [SEL_148] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_147] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query7.q.out ql/src/test/results/clientpositive/perf/query7.q.out index 65478ab..18ae02d 100644 --- ql/src/test/results/clientpositive/perf/query7.q.out +++ ql/src/test/results/clientpositive/perf/query7.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_33] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_30] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(_col4)","avg(_col5)","avg(_col7)","avg(_col6)"],keys:_col18 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col18","_col4","_col5","_col7","_col6"] - Merge Join Operator [MERGEJOIN_58] (rows=843315281 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col18"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_57] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_53] (rows=2300 width=1179) - predicate:(((p_channel_email = 'N') or (p_channel_event = 'N')) and p_promo_sk is not null) - TableScan [TS_9] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_email","p_channel_event"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_56] (rows=696954748 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_52] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=633595212 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_50] (rows=575995635 width=88) - predicate:(ss_cdemo_sk is not null and ss_sold_date_sk is not null and ss_item_sk is not null and ss_promo_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_cdemo_sk","ss_promo_sk","ss_quantity","ss_list_price","ss_sales_price","ss_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_51] (rows=232725 width=385) - predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query70.q.out ql/src/test/results/clientpositive/perf/query70.q.out index d0900a8..b1e4711 100644 --- ql/src/test/results/clientpositive/perf/query70.q.out +++ ql/src/test/results/clientpositive/perf/query70.q.out @@ -75,151 +75,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 12 <- Map 17 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 15 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_64] - Limit [LIM_63] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_62] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_61] - Select Operator [SEL_59] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_58] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_57] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_55] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5"] - Group By Operator [GBY_54] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_52] (rows=2299950717 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col0, _col1, 0 - Select Operator [SEL_50] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_92] (rows=766650239 width=88) - Conds:RS_47._col7=RS_48._col0(Inner),Output:["_col2","_col6","_col7"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Group By Operator [GBY_39] (rows=58079562 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=116159124 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_32] (rows=116159124 width=88) - Output:["_col0"] - Filter Operator [FIL_84] (rows=116159124 width=88) - predicate:(rank_window_0 <= 5) - PTF Operator [PTF_31] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"_col0"}] - Select Operator [SEL_30] (rows=348477374 width=88) - Output:["_col0","_col1"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col6 - Select Operator [SEL_24] (rows=696954748 width=88) - Output:["_col6","_col2"] - Merge Join Operator [MERGEJOIN_91] (rows=696954748 width=88) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col6"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_87] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_state is not null) - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_85] (rows=575995635 width=88) - predicate:(ss_store_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_9] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_86] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_89] (rows=696954748 width=88) - Conds:RS_44._col1=RS_45._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=1704 width=1910) - predicate:(s_state is not null and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,s,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_88] (rows=633595212 width=88) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_81] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_82] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query71.q.out ql/src/test/results/clientpositive/perf/query71.q.out index 9bb010c..cce17b4 100644 --- ql/src/test/results/clientpositive/perf/query71.q.out +++ ql/src/test/results/clientpositive/perf/query71.q.out @@ -5,134 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 15 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Map 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 10 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 3 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_53] - Select Operator [SEL_52] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Select Operator [SEL_49] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_48] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_46] (rows=1341632299 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col0)"],keys:_col4, _col8, _col9, _col5 - Select Operator [SEL_45] (rows=1341632299 width=108) - Output:["_col4","_col8","_col9","_col5","_col0"] - Merge Join Operator [MERGEJOIN_87] (rows=1341632299 width=108) - Conds:RS_42._col2=RS_43._col0(Inner),Output:["_col0","_col4","_col5","_col8","_col9"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=86400 width=471) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=86400 width=471) - predicate:(((t_meal_time = 'breakfast') or (t_meal_time = 'dinner')) and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute","t_meal_time"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_86] (rows=1219665700 width=108) - Conds:Union 3._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_81] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_33] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Union 3 [SIMPLE_EDGE] - <-Reducer 10 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_84] (rows=316788826 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_78] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_77] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null and cs_sold_time_sk is not null) - TableScan [TS_10] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_31] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_85] (rows=633595212 width=88) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_79] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_sold_time_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_80] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_9] (rows=158402938 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_83] (rows=158402938 width=135) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_75] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null and ws_sold_time_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_item_sk","ws_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query72.q.out ql/src/test/results/clientpositive/perf/query72.q.out index 828aa28..45c92f6 100644 --- ql/src/test/results/clientpositive/perf/query72.q.out +++ ql/src/test/results/clientpositive/perf/query72.q.out @@ -61,192 +61,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 22 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 23 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 18 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 20 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 21 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 13 - File Output Operator [FS_76] - Limit [LIM_75] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_74] (rows=41498422 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_73] - Group By Operator [GBY_71] (rows=41498422 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_69] (rows=82996844 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col3)","count(_col4)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_67] (rows=82996844 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Merge Join Operator [MERGEJOIN_143] (rows=82996844 width=135) - Conds:RS_64._col4, _col6=RS_65._col0, _col1(Left Outer),Output:["_col13","_col15","_col22","_col28"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Select Operator [SEL_60] (rows=28798881 width=106) - Output:["_col0","_col1"] - Filter Operator [FIL_133] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_58] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col4, _col6 - Merge Join Operator [MERGEJOIN_142] (rows=75451675 width=135) - Conds:RS_61._col5=RS_62._col0(Left Outer),Output:["_col4","_col6","_col13","_col15","_col22","_col28"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=2300 width=1179) - Output:["_col0"] - TableScan [TS_56] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col5 - Select Operator [SEL_55] (rows=68592431 width=135) - Output:["_col13","_col15","_col22","_col4","_col5","_col6"] - Filter Operator [FIL_54] (rows=68592431 width=135) - predicate:(UDFToDouble(_col27) > (UDFToDouble(_col21) + 5.0)) - Select Operator [SEL_53] (rows=205777294 width=135) - Output:["_col4","_col5","_col6","_col13","_col15","_col21","_col22","_col27"] - Merge Join Operator [MERGEJOIN_141] (rows=205777294 width=135) - Conds:RS_50._col10=RS_51._col0(Inner),Output:["_col4","_col5","_col6","_col13","_col14","_col23","_col25","_col27"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_131] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_29] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col10 - Merge Join Operator [MERGEJOIN_140] (rows=187070264 width=135) - Conds:RS_47._col1=RS_48._col0(Inner),Output:["_col4","_col5","_col6","_col10","_col13","_col14","_col23","_col25"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_130] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_26] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_139] (rows=170063873 width=135) - Conds:RS_44._col4=RS_45._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col10","_col13","_col14","_col23"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_129] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_23] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_138] (rows=154603518 width=135) - Conds:RS_41._col3=RS_42._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_128] (rows=3600 width=107) - predicate:((hd_buy_potential = '1001-5000') and hd_demo_sk is not null) - TableScan [TS_20] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_137] (rows=140548650 width=135) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=930900 width=385) - Output:["_col0"] - Filter Operator [FIL_127] (rows=930900 width=385) - predicate:((cd_marital_status = 'M') and cd_demo_sk is not null) - TableScan [TS_17] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_136] (rows=127771498 width=135) - Conds:RS_35._col8, _col14=RS_36._col0, _col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Select Operator [SEL_16] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_week_seq is not null) - TableScan [TS_14] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col8, _col14 - Merge Join Operator [MERGEJOIN_135] (rows=116155905 width=135) - Conds:RS_32._col0=RS_33._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col10","_col13","_col14"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=36524 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_125] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null and d_week_seq is not null) - TableScan [TS_11] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_week_seq","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=105596275 width=135) - Output:["_col0","_col1","_col10","_col2","_col3","_col4","_col5","_col6","_col8"] - Filter Operator [FIL_9] (rows=105596275 width=135) - predicate:(_col11 < _col7) - Merge Join Operator [MERGEJOIN_134] (rows=316788826 width=135) - Conds:RS_6._col4=RS_7._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col4 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_sold_date_sk is not null and cs_ship_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_date_sk","cs_bill_cdemo_sk","cs_bill_hdemo_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_quantity"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_124] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_3] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] + Vertex 13 diff --git ql/src/test/results/clientpositive/perf/query73.q.out ql/src/test/results/clientpositive/perf/query73.q.out index 9e1a39a..5daf3d9 100644 --- ql/src/test/results/clientpositive/perf/query73.q.out +++ ql/src/test/results/clientpositive/perf/query73.q.out @@ -5,96 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=88000001 width=860) - Conds:RS_31._col0=RS_32._col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Filter Operator [FIL_29] (rows=42591679 width=88) - predicate:_col2 BETWEEN 1 AND 5 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_56] (rows=852 width=1910) - predicate:((s_county) IN ('Kittitas County', 'Adams County', 'Richland County', 'Furnas County') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1200 width=107) - Output:["_col0"] - Filter Operator [FIL_55] (rows=1200 width=107) - predicate:(((hd_buy_potential = '1001-5000') or (hd_buy_potential = '5001-10000')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.0)) ELSE (null) END and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential","hd_dep_count","hd_vehicle_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_store_sk","ss_ticket_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=4058 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query75.q.out ql/src/test/results/clientpositive/perf/query75.q.out index f78ce8a..b93f79e 100644 --- ql/src/test/results/clientpositive/perf/query75.q.out +++ ql/src/test/results/clientpositive/perf/query75.q.out @@ -5,387 +5,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 21 <- Map 24 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Map 25 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) -Reducer 28 <- Map 33 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 34 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Union 30 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Map 40 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Map 41 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 44 <- Map 47 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 45 <- Map 48 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 31 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 5 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 22 <- Union 5 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 30 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 38 <- Union 30 (CONTAINS), Vertex 37 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 45 <- Union 30 (CONTAINS), Vertex 44 (SORT_PARTITION_EDGE), Vertex 48 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_156] - Limit [LIM_155] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_154] (rows=245965926 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_153] - Select Operator [SEL_152] (rows=245965926 width=108) - Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_151] (rows=245965926 width=108) - predicate:((CAST( _col4 AS decimal(17,2)) / CAST( _col10 AS decimal(17,2))) < 0.9) - Merge Join Operator [MERGEJOIN_259] (rows=737897778 width=108) - Conds:RS_148._col0, _col1, _col2, _col3=RS_149._col0, _col1, _col2, _col3(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col10","_col11"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_146] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Union 30 [SIMPLE_EDGE] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_95] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_252] (rows=383314495 width=135) - Conds:RS_92._col1, _col2=RS_93._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_232] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_251] (rows=348467716 width=135) - Conds:RS_89._col1=RS_90._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_82] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_231] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_80] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_250] (rows=316788826 width=135) - Conds:RS_86._col0=RS_87._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Select Operator [SEL_76] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_229] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_74] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_230] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 38 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_117] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_255] (rows=766650239 width=88) - Conds:RS_114._col1, _col2=RS_115._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_236] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_105] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_254] (rows=696954748 width=88) - Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_104] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_235] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_102] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_253] (rows=633595212 width=88) - Conds:RS_108._col0=RS_109._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_233] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_96] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_234] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_99] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 45 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_141] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_258] (rows=191667562 width=135) - Conds:RS_138._col1, _col2=RS_139._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_139] - PartitionCols:_col0, _col1 - Select Operator [SEL_131] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_240] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_129] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_138] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_257] (rows=174243235 width=135) - Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_136] - PartitionCols:_col0 - Select Operator [SEL_128] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_239] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_126] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_256] (rows=158402938 width=135) - Conds:RS_132._col0=RS_133._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col0 - Select Operator [SEL_122] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_237] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_120] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Select Operator [SEL_125] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_238] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_123] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_72] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_43] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_246] (rows=766650239 width=88) - Conds:RS_40._col1, _col2=RS_41._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Select Operator [SEL_33] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_224] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_31] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_245] (rows=696954748 width=88) - Conds:RS_37._col1=RS_38._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_30] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_223] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_28] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_244] (rows=633595212 width=88) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_221] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_222] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_67] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_249] (rows=191667562 width=135) - Conds:RS_64._col1, _col2=RS_65._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Select Operator [SEL_57] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_228] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_55] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_248] (rows=174243235 width=135) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_227] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_52] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_247] (rows=158402938 width=135) - Conds:RS_58._col0=RS_59._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_225] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_46] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_226] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_49] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_21] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_243] (rows=383314495 width=135) - Conds:RS_18._col1, _col2=RS_19._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_220] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_9] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_242] (rows=348467716 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_219] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_241] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_217] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_218] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query76.q.out ql/src/test/results/clientpositive/perf/query76.q.out index c7dbb37..8f1d85c 100644 --- ql/src/test/results/clientpositive/perf/query76.q.out +++ ql/src/test/results/clientpositive/perf/query76.q.out @@ -5,151 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 16 <- Map 18 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 4 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_59] - Limit [LIM_58] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_57] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Group By Operator [GBY_54] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_31] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_87] (rows=87121617 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col3","_col5","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_80] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_22] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=79201469 width=135) - Conds:RS_25._col1=RS_26._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_19] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col1 - Select Operator [SEL_18] (rows=72001334 width=135) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_78] (rows=72001334 width=135) - predicate:(ws_web_page_sk is null and ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_16] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_page_sk","ws_ext_sales_price"] - <-Reducer 16 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_49] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_89] (rows=174233858 width=135) - Conds:RS_46._col0=RS_47._col0(Inner),Output:["_col3","_col5","_col7","_col8"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_40] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_88] (rows=158394413 width=135) - Conds:RS_43._col2=RS_44._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col2 - Select Operator [SEL_36] (rows=143994918 width=135) - Output:["_col0","_col2","_col3"] - Filter Operator [FIL_81] (rows=143994918 width=135) - predicate:(cs_warehouse_sk is null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_34] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_warehouse_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_82] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Reducer 3 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_15] (rows=348477373 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_85] (rows=348477373 width=88) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col1","_col5","_col7","_col8"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_77] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_84] (rows=316797605 width=88) - Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=287997817 width=88) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_76] (rows=287997817 width=88) - predicate:(ss_addr_sk is null and ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query79.q.out ql/src/test/results/clientpositive/perf/query79.q.out index 6ce9592..fc79760 100644 --- ql/src/test/results/clientpositive/perf/query79.q.out +++ ql/src/test/results/clientpositive/perf/query79.q.out @@ -5,98 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_35] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=421657640 width=88) - Conds:RS_30._col0=RS_31._col1(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col6)","sum(_col7)"],keys:_col1, _col3, _col5, _col13 - Select Operator [SEL_24] (rows=766650239 width=88) - Output:["_col1","_col3","_col5","_col13","_col6","_col7"] - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col13"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_56] (rows=6000 width=107) - predicate:(((hd_dep_count = 8) or (hd_vehicle_count > 0)) and hd_demo_sk is not null) - TableScan [TS_12] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col4=RS_19._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7","_col13"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=189 width=1910) - Output:["_col0","_col2"] - Filter Operator [FIL_55] (rows=189 width=1910) - predicate:(s_number_employees BETWEEN 200 AND 295 and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_number_employees","s_city"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_coupon_amt","ss_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=18262 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and (d_dow = 1) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query8.q.out ql/src/test/results/clientpositive/perf/query8.q.out index 5642336..fdd39bf 100644 --- ql/src/test/results/clientpositive/perf/query8.q.out +++ ql/src/test/results/clientpositive/perf/query8.q.out @@ -215,146 +215,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Union 9 (SIMPLE_EDGE) -Reducer 11 <- Map 17 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 9 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 9 (CONTAINS) +Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 9 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 9 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_63] - Limit [LIM_62] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_61] (rows=348477374 width=88) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Group By Operator [GBY_58] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Group By Operator [GBY_56] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col8 - Select Operator [SEL_55] (rows=696954748 width=88) - Output:["_col8","_col2"] - Merge Join Operator [MERGEJOIN_89] (rows=696954748 width=88) - Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2","_col8"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_88] (rows=1874 width=1911) - Conds:RS_45.substr(_col0, 1, 2)=RS_46.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:substr(_col2, 1, 2) - Select Operator [SEL_44] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_85] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_zip is not null) - TableScan [TS_42] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:substr(_col0, 1, 2) - Select Operator [SEL_41] (rows=1 width=1014) - Output:["_col0"] - Filter Operator [FIL_40] (rows=1 width=1014) - predicate:(_col1 = 2) - Group By Operator [GBY_39] (rows=6833333 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Union 9 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=13666666 width=1014) - Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 - Group By Operator [GBY_32] (rows=3666666 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=7333333 width=1014) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_28] (rows=7333333 width=1014) - Output:["_col0"] - Filter Operator [FIL_27] (rows=7333333 width=1014) - predicate:(_col1 > 10) - Group By Operator [GBY_26] (rows=22000000 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=44000000 width=1014) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=44000000 width=1014) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_83] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and substr(ca_zip, 1, 5) is not null) - TableScan [TS_14] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_zip"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=40000000 width=860) - Output:["_col0"] - Filter Operator [FIL_84] (rows=40000000 width=860) - predicate:((c_preferred_cust_flag = 'Y') and c_current_addr_sk is not null) - TableScan [TS_17] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_current_addr_sk","c_preferred_cust_flag"] - <-Reducer 8 [CONTAINS] - Reduce Output Operator [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=13666666 width=1014) - Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 - Group By Operator [GBY_12] (rows=10000000 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=20000000 width=1014) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_81] (rows=20000000 width=1014) - predicate:(substr(ca_zip, 1, 5)) IN ('89436', '30868', '65085', '22977', '83927', '77557', '58429', '40697', '80614', '10502', '32779', '91137', '61265', '98294', '17921', '18427', '21203', '59362', '87291', '84093', '21505', '17184', '10866', '67898', '25797', '28055', '18377', '80332', '74535', '21757', '29742', '90885', '29898', '17819', '40811', '25990', '47513', '89531', '91068', '10391', '18846', '99223', '82637', '41368', '83658', '86199', '81625', '26696', '89338', '88425', '32200', '81427', '19053', '77471', '36610', '99823', '43276', '41249', '48584', '83550', '82276', '18842', '78890', '14090', '38123', '40936', '34425', '19850', '43286', '80072', '79188', '54191', '11395', '50497', '84861', '90733', '21068', '57666', '37119', '25004', '57835', '70067', '62878', '95806', '19303', '18840', '19124', '29785', '16737', '16022', '49613', '89977', '68310', '60069', '98360', '48649', '39050', '41793', '25002', '27413', '39736', '47208', '16515', '94808', '57648', '15009', '80015', '42961', '63982', '21744', '71853', '81087', '67468', '34175', '64008', '20261', '11201', '51799', '48043', '45645', '61163', '48375', '36447', '57042', '21218', '41100', '89951', '22745', '35851', '83326', '61125', '78298', '80752', '49858', '52940', '96976', '63792', '11376', '53582', '18717', '90226', '50530', '94203', '99447', '27670', '96577', '57856', '56372', '16165', '23427', '54561', '28806', '44439', '22926', '30123', '61451', '92397', '56979', '92309', '70873', '13355', '21801', '46346', '37562', '56458', '28286', '47306', '99555', '69399', '26234', '47546', '49661', '88601', '35943', '39936', '25632', '24611', '44166', '56648', '30379', '59785', '11110', '14329', '93815', '52226', '71381', '13842', '25612', '63294', '14664', '21077', '82626', '18799', '60915', '81020', '56447', '76619', '11433', '13414', '42548', '92713', '70467', '30884', '47484', '16072', '38936', '13036', '88376', '45539', '35901', '19506', '65690', '73957', '71850', '49231', '14276', '20005', '18384', '76615', '11635', '38177', '55607', '41369', '95447', '58581', '58149', '91946', '33790', '76232', '75692', '95464', '22246', '51061', '56692', '53121', '77209', '15482', '10688', '14868', '45907', '73520', '72666', '25734', '17959', '24677', '66446', '94627', '53535', '15560', '41967', '69297', '11929', '59403', '33283', '52232', '57350', '43933', '40921', '36635', '10827', '71286', '19736', '80619', '25251', '95042', '15526', '36496', '55854', '49124', '81980', '35375', '49157', '63512', '28944', '14946', '36503', '54010', '18767', '23969', '43905', '66979', '33113', '21286', '58471', '59080', '13395', '79144', '70373', '67031', '38360', '26705', '50906', '52406', '26066', '73146', '15884', '31897', '30045', '61068', '45550', '92454', '13376', '14354', '19770', '22928', '97790', '50723', '46081', '30202', '14410', '20223', '88500', '67298', '13261', '14172', '81410', '93578', '83583', '46047', '94167', '82564', '21156', '15799', '86709', '37931', '74703', '83103', '23054', '70470', '72008', '49247', '91911', '69998', '20961', '70070', '63197', '54853', '88191', '91830', '49521', '19454', '81450', '89091', '62378', '25683', '61869', '51744', '36580', '85778', '36871', '48121', '28810', '83712', '45486', '67393', '26935', '42393', '20132', '55349', '86057', '21309', '80218', '10094', '11357', '48819', '39734', '40758', '30432', '21204', '29467', '30214', '61024', '55307', '74621', '11622', '68908', '33032', '52868', '99194', '99900', '84936', '69036', '99149', '45013', '32895', '59004', '32322', '14933', '32936', '33562', '72550', '27385', '58049', '58200', '16808', '21360', '32961', '18586', '79307', '15492') - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_zip"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_86] (rows=633595212 width=88) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_79] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query80.q.out ql/src/test/results/clientpositive/perf/query80.q.out index 7338d9c..3f73b54 100644 --- ql/src/test/results/clientpositive/perf/query80.q.out +++ ql/src/test/results/clientpositive/perf/query80.q.out @@ -191,318 +191,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 18 <- Map 24 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 25 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE) -Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 27 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 36 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE) -Reducer 31 <- Map 37 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 38 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 33 <- Map 39 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 34 <- Reducer 33 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 11 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 22 <- Union 8 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 34 <- Union 8 (CONTAINS), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 8 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 10 - File Output Operator [FS_128] - Limit [LIM_127] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_126] (rows=1217531358 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_125] - Select Operator [SEL_124] (rows=1217531358 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_123] (rows=1217531358 width=108) - Output:["_col0","_col1","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_77] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_76] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Group By Operator [GBY_74] (rows=463810558 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_72] (rows=463810558 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_213] (rows=463810558 width=135) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=46000 width=460) - Output:["_col0","_col1"] - Filter Operator [FIL_197] (rows=46000 width=460) - predicate:cp_catalog_page_sk is not null - TableScan [TS_54] (rows=46000 width=460) - default@catalog_page,catalog_page,Tbl:COMPLETE,Col:NONE,Output:["cp_catalog_page_sk","cp_catalog_page_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_212] (rows=421645953 width=135) - Conds:RS_66._col3=RS_67._col0(Inner),Output:["_col1","_col5","_col6","_col9","_col10"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_196] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_51] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_211] (rows=383314495 width=135) - Conds:RS_63._col2=RS_64._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col9","_col10"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_195] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_48] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_210] (rows=348467716 width=135) - Conds:RS_60._col0=RS_61._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_194] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - TableScan [TS_45] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_209] (rows=316788826 width=135) - Conds:RS_57._col2, _col4=RS_58._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col2, _col4 - Select Operator [SEL_41] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_192] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_catalog_page_sk is not null and cs_item_sk is not null and cs_promo_sk is not null) - TableScan [TS_39] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_catalog_page_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_ext_sales_price","cs_net_profit"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0, _col1 - Select Operator [SEL_44] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_193] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_42] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_amount","cr_net_loss"] - <-Reducer 34 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_118] (rows=115958879 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_117] (rows=115958879 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=231917759 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_113] (rows=231917759 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_218] (rows=231917759 width=135) - Conds:RS_110._col2=RS_111._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col0 - Select Operator [SEL_97] (rows=84 width=1850) - Output:["_col0","_col1"] - Filter Operator [FIL_203] (rows=84 width=1850) - predicate:web_site_sk is not null - TableScan [TS_95] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_site_id"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_217] (rows=210834322 width=135) - Conds:RS_107._col3=RS_108._col0(Inner),Output:["_col2","_col5","_col6","_col9","_col10"] - <-Map 38 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_202] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_92] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_216] (rows=191667562 width=135) - Conds:RS_104._col1=RS_105._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_201] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_89] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_215] (rows=174243235 width=135) - Conds:RS_101._col0=RS_102._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_200] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - TableScan [TS_86] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_214] (rows=158402938 width=135) - Conds:RS_98._col1, _col4=RS_99._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col1, _col4 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_198] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_web_site_sk is not null and ws_item_sk is not null and ws_promo_sk is not null) - TableScan [TS_80] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_site_sk","ws_promo_sk","ws_order_number","ws_ext_sales_price","ws_net_profit"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_199] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_83] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_amt","wr_net_loss"] - <-Reducer 7 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_38] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_37] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Group By Operator [GBY_35] (rows=927646829 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_33] (rows=927646829 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_208] (rows=927646829 width=88) - Conds:RS_30._col2=RS_31._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_207] (rows=843315281 width=88) - Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col2","_col5","_col6","_col9","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_190] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_12] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_206] (rows=766650239 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_189] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_205] (rows=696954748 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_188] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_204] (rows=633595212 width=88) - Conds:RS_18._col1, _col4=RS_19._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1, _col4 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_186] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null and ss_promo_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_ext_sales_price","ss_net_profit"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_187] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_3] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_amt","sr_net_loss"] + Vertex 10 diff --git ql/src/test/results/clientpositive/perf/query81.q.out ql/src/test/results/clientpositive/perf/query81.q.out index 8ca0068..1810bcd 100644 --- ql/src/test/results/clientpositive/perf/query81.q.out +++ ql/src/test/results/clientpositive/perf/query81.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 22' is a cross product +Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 22' is a cross product PREHOOK: query: explain with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, @@ -60,249 +60,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE), Map 30 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 26 <- Map 29 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_102] - Limit [LIM_101] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_100] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_99] - Select Operator [SEL_98] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col11","_col12","_col13","_col14","_col15","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_97] (rows=930023387364950016 width=1) - predicate:(_col2 > CASE WHEN (_col22 is null) THEN (null) ELSE (_col21) END) - Select Operator [SEL_96] (rows=2790070162094850048 width=1) - Output:["_col2","_col4","_col5","_col6","_col7","_col8","_col9","_col11","_col12","_col13","_col14","_col16","_col18","_col19","_col20","_col21","_col22"] - Merge Join Operator [MERGEJOIN_162] (rows=2790070162094850048 width=1) - Conds:RS_93._col19=RS_94._col2(Left Outer),Output:["_col1","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col14","_col15","_col16","_col17","_col20","_col21","_col22"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col2 - Select Operator [SEL_86] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_85] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col3 - Select Operator [SEL_82] (rows=5072854730221289472 width=1) - Output:["_col3","_col2"] - Merge Join Operator [MERGEJOIN_161] (rows=5072854730221289472 width=1) - Conds:RS_79._col1=RS_80._col0(Inner),Output:["_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=22000000 width=1014) - Output:["_col1","_col2"] - Group By Operator [GBY_44] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1 - Group By Operator [GBY_42] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col7, _col1 - Select Operator [SEL_41] (rows=44000000 width=1014) - Output:["_col7","_col1","_col3"] - Merge Join Operator [MERGEJOIN_159] (rows=44000000 width=1014) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_32] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_158] (rows=31678769 width=106) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_146] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_returning_addr_sk is not null) - TableScan [TS_26] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk","cr_return_amt_inc_tax"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_147] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_77] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Group By Operator [GBY_75] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_154] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_70] - Select Operator [SEL_47] (rows=80000000 width=4) - TableScan [TS_46] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:COMPLETE - <-Map 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_72] - Select Operator [SEL_69] (rows=40000000 width=4) - TableScan [TS_68] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_67] (rows=22000000 width=1014) - Output:["_col1"] - Group By Operator [GBY_66] (rows=22000000 width=1014) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Group By Operator [GBY_64] (rows=44000000 width=1014) - Output:["_col0","_col1"],keys:_col7, _col1 - Select Operator [SEL_63] (rows=44000000 width=1014) - Output:["_col7","_col1"] - Merge Join Operator [MERGEJOIN_153] (rows=44000000 width=1014) - Conds:RS_60._col2=RS_61._col0(Inner),Output:["_col1","_col7"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_54] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_152] (rows=31678769 width=106) - Conds:RS_57._col0=RS_58._col0(Inner),Output:["_col1","_col2"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_149] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_returning_addr_sk is not null) - TableScan [TS_48] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_150] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_51] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col19 - Merge Join Operator [MERGEJOIN_160] (rows=96800003 width=860) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col14","_col15","_col16","_col17","_col19","_col20"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1 - Group By Operator [GBY_22] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col1, _col7 - Select Operator [SEL_21] (rows=44000000 width=1014) - Output:["_col1","_col7","_col3"] - Merge Join Operator [MERGEJOIN_157] (rows=44000000 width=1014) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_12] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_156] (rows=31678769 width=106) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_144] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_143] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_returning_addr_sk is not null and cr_returning_customer_sk is not null) - TableScan [TS_6] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk","cr_return_amt_inc_tax"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_155] (rows=88000001 width=860) - Conds:RS_87._col2=RS_88._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col14","_col15","_col16","_col17"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_141] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_current_addr_sk","c_salutation","c_first_name","c_last_name"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0","_col1","_col10","_col11","_col2","_col3","_col4","_col5","_col6","_col8","_col9"] - Filter Operator [FIL_142] (rows=20000000 width=1014) - predicate:((ca_state = 'IL') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_street_type","ca_suite_number","ca_city","ca_county","ca_state","ca_zip","ca_country","ca_gmt_offset","ca_location_type"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query82.q.out ql/src/test/results/clientpositive/perf/query82.q.out index 9c0042c..271a324 100644 --- ql/src/test/results/clientpositive/perf/query82.q.out +++ ql/src/test/results/clientpositive/perf/query82.q.out @@ -33,71 +33,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_26] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_23] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_21] (rows=1267190424 width=88) - Output:["_col0","_col1","_col2"],keys:_col2, _col3, _col4 - Merge Join Operator [MERGEJOIN_44] (rows=1267190424 width=88) - Conds:RS_16._col0=RS_17._col0(Inner),RS_17._col0=RS_18._col1(Inner),Output:["_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0"] - Filter Operator [FIL_39] (rows=575995635 width=88) - predicate:ss_item_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25666 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=25666 width=1436) - predicate:((i_manufact_id) IN (437, 129, 727, 663) and i_current_price BETWEEN 30 AND 60 and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_manufact_id"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=4593600 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4176000 width=15) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=4176000 width=15) - predicate:(inv_quantity_on_hand BETWEEN 100 AND 500 and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_6] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2002-05-30 AND 2002-07-29 00:00:00.0 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query83.q.out ql/src/test/results/clientpositive/perf/query83.q.out index 9960bc7..0ef2bb8 100644 --- ql/src/test/results/clientpositive/perf/query83.q.out +++ ql/src/test/results/clientpositive/perf/query83.q.out @@ -131,308 +131,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 21 <- Map 20 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE) -Reducer 28 <- Map 27 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 36 <- Reducer 35 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 18 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_134] - Limit [LIM_133] (rows=100 width=77) - Number of rows:100 - Select Operator [SEL_132] (rows=76653825 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_131] - Select Operator [SEL_130] (rows=76653825 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_228] (rows=76653825 width=77) - Conds:RS_126._col0=RS_127._col0(Inner),RS_126._col0=RS_128._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_127] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=34842647 width=77) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Group By Operator [GBY_80] (rows=69685294 width=77) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 - Select Operator [SEL_79] (rows=69685294 width=77) - Output:["_col4","_col2"] - Merge Join Operator [MERGEJOIN_226] (rows=69685294 width=77) - Conds:RS_76._col0=RS_77._col0(Inner),Output:["_col2","_col4"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_219] (rows=63350266 width=77) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Select Operator [SEL_44] (rows=57591150 width=77) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_206] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_returned_date_sk is not null) - TableScan [TS_42] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_return_quantity"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_207] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_45] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_221] (rows=80353 width=1119) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col0"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Select Operator [SEL_50] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_208] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - TableScan [TS_48] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Group By Operator [GBY_67] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_65] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_220] (rows=80353 width=1119) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col0"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_53] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_209] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_51] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Group By Operator [GBY_59] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_56] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_210] (rows=36525 width=1119) - predicate:((d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') and d_week_seq is not null) - TableScan [TS_54] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_128] - PartitionCols:_col0 - Group By Operator [GBY_124] (rows=8711072 width=92) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_123] - PartitionCols:_col0 - Group By Operator [GBY_122] (rows=17422145 width=92) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 - Select Operator [SEL_121] (rows=17422145 width=92) - Output:["_col4","_col2"] - Merge Join Operator [MERGEJOIN_227] (rows=17422145 width=92) - Conds:RS_118._col0=RS_119._col0(Inner),Output:["_col2","_col4"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_222] (rows=15838314 width=92) - Conds:RS_115._col1=RS_116._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col1 - Select Operator [SEL_86] (rows=14398467 width=92) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_211] (rows=14398467 width=92) - predicate:(wr_item_sk is not null and wr_returned_date_sk is not null) - TableScan [TS_84] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_return_quantity"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_212] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_87] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_224] (rows=80353 width=1119) - Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col0"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col1 - Select Operator [SEL_92] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_213] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - TableScan [TS_90] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Group By Operator [GBY_107] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_223] (rows=80353 width=1119) - Conds:RS_103._col1=RS_104._col0(Inner),Output:["_col0"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col1 - Select Operator [SEL_95] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_214] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_93] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Group By Operator [GBY_101] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_99] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_98] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_215] (rows=36525 width=1119) - predicate:((d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') and d_week_seq is not null) - TableScan [TS_96] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Group By Operator [GBY_40] (rows=17423323 width=106) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=34846646 width=106) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 - Select Operator [SEL_37] (rows=34846646 width=106) - Output:["_col4","_col2"] - Merge Join Operator [MERGEJOIN_225] (rows=34846646 width=106) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col2","_col4"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_216] (rows=31678769 width=106) - Conds:RS_31._col1=RS_32._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_201] (rows=28798881 width=106) - predicate:(cr_item_sk is not null and cr_returned_date_sk is not null) - TableScan [TS_0] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_item_sk","cr_return_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_202] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_218] (rows=80353 width=1119) - Conds:RS_27._col1=RS_28._col0(Inner),Output:["_col0"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_203] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_217] (rows=80353 width=1119) - Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_204] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_14] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_205] (rows=36525 width=1119) - predicate:((d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') and d_week_seq is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query84.q.out ql/src/test/results/clientpositive/perf/query84.q.out index cf68729..8ccd524 100644 --- ql/src/test/results/clientpositive/perf/query84.q.out +++ ql/src/test/results/clientpositive/perf/query84.q.out @@ -5,95 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 7 <- Reducer 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Vertex 10 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=860) - Number of rows:100 - Select Operator [SEL_35] (rows=212960011 width=860) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=212960011 width=860) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_67] (rows=212960011 width=860) - Conds:RS_29._col0=RS_30._col0(Inner),RS_30._col0=RS_31._col1(Inner),Output:["_col2","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0"] - Filter Operator [FIL_58] (rows=57591150 width=77) - predicate:sr_cdemo_sk is not null - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_cdemo_sk"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_59] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_66] (rows=96800003 width=860) - Conds:RS_25._col2=RS_26._col0(Inner),Output:["_col0","_col1","_col4","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_65] (rows=7920 width=107) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=2 width=12) - Output:["_col0"] - Filter Operator [FIL_63] (rows=2 width=12) - predicate:((ib_lower_bound >= 32287) and (ib_upper_bound <= 82287) and ib_income_band_sk is not null) - TableScan [TS_15] (rows=20 width=12) - default@income_band,income_band,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk","ib_lower_bound","ib_upper_bound"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_12] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_64] (rows=88000001 width=860) - Conds:RS_22._col3=RS_23._col0(Inner),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col3 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_60] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null) - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_id","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_61] (rows=20000000 width=1014) - predicate:((ca_city = 'Hopewell') and ca_address_sk is not null) - TableScan [TS_9] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query85.q.out ql/src/test/results/clientpositive/perf/query85.q.out index e7f7348..45918ea 100644 --- ql/src/test/results/clientpositive/perf/query85.q.out +++ ql/src/test/results/clientpositive/perf/query85.q.out @@ -5,153 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Map 17 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 8 <- Map 16 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_59] - Limit [LIM_58] (rows=100 width=1014) - Number of rows:100 - Select Operator [SEL_57] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Select Operator [SEL_55] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_54] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Group By Operator [GBY_52] (rows=2440165 width=1014) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col6)","avg(_col16)","avg(_col15)"],keys:_col28 - Select Operator [SEL_51] (rows=2440165 width=1014) - Output:["_col28","_col6","_col16","_col15"] - Merge Join Operator [MERGEJOIN_109] (rows=2440165 width=1014) - Conds:RS_48._col13=RS_49._col0(Inner),Output:["_col6","_col15","_col16","_col28"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=72 width=200) - Output:["_col0","_col1"] - Filter Operator [FIL_102] (rows=72 width=200) - predicate:r_reason_sk is not null - TableScan [TS_42] (rows=72 width=200) - default@reason,reason,Tbl:COMPLETE,Col:NONE,Output:["r_reason_sk","r_reason_desc"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col13 - Merge Join Operator [MERGEJOIN_108] (rows=2218332 width=1014) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col6","_col13","_col15","_col16"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_95] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=2016666 width=1014) - Output:["_col0","_col11","_col13","_col14","_col4"] - Filter Operator [FIL_40] (rows=2016666 width=1014) - predicate:(((_col23) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 100 AND 200) or ((_col23) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 300) or ((_col23) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 250)) - Select Operator [SEL_39] (rows=12100000 width=1014) - Output:["_col0","_col4","_col6","_col11","_col13","_col14","_col23"] - Merge Join Operator [MERGEJOIN_107] (rows=12100000 width=1014) - Conds:RS_36._col13, _col20, _col21=RS_37._col0, _col1, _col2(Inner),Output:["_col1","_col3","_col7","_col9","_col14","_col16","_col17"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_32] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_101] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - TableScan [TS_30] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col13, _col20, _col21 - Merge Join Operator [MERGEJOIN_106] (rows=11000000 width=1014) - Conds:RS_33._col0=RS_34._col9(Inner),Output:["_col1","_col3","_col7","_col9","_col13","_col14","_col16","_col17","_col20","_col21"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col9 - Select Operator [SEL_29] (rows=1774698 width=135) - Output:["_col0","_col10","_col11","_col13","_col14","_col17","_col18","_col4","_col6","_col9"] - Filter Operator [FIL_28] (rows=1774698 width=135) - predicate:(((_col17 = 'M') and (_col18 = '4 yr Degree') and _col5 BETWEEN 100 AND 150) or ((_col17 = 'D') and (_col18 = 'Primary') and _col5 BETWEEN 50 AND 100) or ((_col17 = 'U') and (_col18 = 'Advanced Degree') and _col5 BETWEEN 150 AND 200)) - Select Operator [SEL_27] (rows=21296393 width=135) - Output:["_col0","_col4","_col5","_col6","_col9","_col10","_col11","_col13","_col14","_col17","_col18"] - Merge Join Operator [MERGEJOIN_105] (rows=21296393 width=135) - Conds:RS_24._col9=RS_25._col0(Inner),Output:["_col1","_col5","_col6","_col7","_col10","_col11","_col12","_col14","_col15","_col17","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_100] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - TableScan [TS_15] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_104] (rows=19360357 width=135) - Conds:RS_21._col2, _col4=RS_22._col0, _col5(Inner),Output:["_col1","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col14","_col15"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col5 - Select Operator [SEL_14] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_99] (rows=14398467 width=92) - predicate:(wr_item_sk is not null and wr_order_number is not null and wr_refunded_cdemo_sk is not null and wr_returning_cdemo_sk is not null and wr_refunded_addr_sk is not null and wr_reason_sk is not null) - TableScan [TS_12] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_refunded_cdemo_sk","wr_refunded_addr_sk","wr_returning_cdemo_sk","wr_reason_sk","wr_order_number","wr_fee","wr_refunded_cash"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2, _col4 - Merge Join Operator [MERGEJOIN_103] (rows=17600325 width=135) - Conds:RS_18._col0=RS_19._col2(Inner),Output:["_col1","_col2","_col4","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col2 - Select Operator [SEL_11] (rows=16000296 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_98] (rows=16000296 width=135) - predicate:((ws_sales_price BETWEEN 100 AND 150 or ws_sales_price BETWEEN 50 AND 100 or ws_sales_price BETWEEN 150 AND 200) and (ws_net_profit BETWEEN 100 AND 200 or ws_net_profit BETWEEN 150 AND 300 or ws_net_profit BETWEEN 50 AND 250) and ws_order_number is not null and ws_item_sk is not null and ws_web_page_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_9] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_page_sk","ws_order_number","ws_quantity","ws_sales_price","ws_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4602 width=585) - Output:["_col0"] - Filter Operator [FIL_97] (rows=4602 width=585) - predicate:wp_web_page_sk is not null - TableScan [TS_6] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query86.q.out ql/src/test/results/clientpositive/perf/query86.q.out index 6377c43..8e10e58 100644 --- ql/src/test/results/clientpositive/perf/query86.q.out +++ ql/src/test/results/clientpositive/perf/query86.q.out @@ -51,76 +51,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_29] - Limit [LIM_28] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_27] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - Select Operator [SEL_24] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_23] (rows=261364852 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_22] (rows=261364852 width=135) - Output:["_col0","_col1","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_20] (rows=261364852 width=135) - Output:["_col0","_col1","_col4","_col5"] - Group By Operator [GBY_19] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_17] (rows=522729705 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col0, _col1, 0 - Select Operator [SEL_15] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_39] (rows=174243235 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_38] (rows=158402938 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_35] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_net_paid"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_36] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query87.q.out ql/src/test/results/clientpositive/perf/query87.q.out index 09f1539..8d752e4 100644 --- ql/src/test/results/clientpositive/perf/query87.q.out +++ ql/src/test/results/clientpositive/perf/query87.q.out @@ -43,217 +43,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 20 <- Map 23 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Reducer 20 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 5 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 21 <- Union 7 (CONTAINS), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_104] - Group By Operator [GBY_102] (rows=1 width=24) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 8 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_101] - Group By Operator [GBY_100] (rows=1 width=24) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_98] (rows=4537552 width=129) - Filter Operator [FIL_97] (rows=4537552 width=129) - predicate:((_col3 > 0) and ((_col3 * 2) = _col4)) - Select Operator [SEL_129] (rows=27225312 width=129) - Output:["_col3","_col4"] - Group By Operator [GBY_96] (rows=27225312 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 21 [CONTAINS] - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_94] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_92] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_90] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_89] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_83] (rows=87121617 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_80] (rows=174243235 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_135] (rows=174243235 width=135) - Conds:RS_77._col1=RS_78._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_78] - PartitionCols:_col0 - Select Operator [SEL_73] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_128] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_71] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_134] (rows=158402938 width=135) - Conds:RS_74._col0=RS_75._col0(Inner),Output:["_col1","_col3"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_67] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_65] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Select Operator [SEL_70] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_127] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_68] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_94] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_92] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_64] (rows=10889817 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_63] (rows=10889817 width=103) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(2)"],keys:_col0, _col1, _col2 - Select Operator [SEL_59] (rows=21779634 width=103) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_58] (rows=21779634 width=103) - predicate:((_col3 > 0) and ((_col3 * 2) = _col4)) - Group By Operator [GBY_57] (rows=130677808 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_55] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_53] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_51] (rows=87116929 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_50] (rows=87116929 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_44] (rows=174233858 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_42] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_41] (rows=348467716 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_133] (rows=348467716 width=135) - Conds:RS_38._col1=RS_39._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_125] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_32] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_132] (rows=316788826 width=135) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_26] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_124] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_55] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_53] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_25] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(2)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_131] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_122] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_130] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_120] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_customer_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_121] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 9 diff --git ql/src/test/results/clientpositive/perf/query88.q.out ql/src/test/results/clientpositive/perf/query88.q.out index 5072997..b18808f 100644 --- ql/src/test/results/clientpositive/perf/query88.q.out +++ ql/src/test/results/clientpositive/perf/query88.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select * from @@ -186,496 +186,43 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 24 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 25 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (CUSTOM_SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (CUSTOM_SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 36 <- Map 40 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) -Reducer 37 <- Map 41 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 47 (SIMPLE_EDGE) -Reducer 44 <- Map 48 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 45 <- Map 49 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE) -Reducer 46 <- Reducer 45 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 51 <- Map 50 (SIMPLE_EDGE), Map 55 (SIMPLE_EDGE) -Reducer 52 <- Map 56 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) -Reducer 53 <- Map 57 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 54 <- Reducer 53 (CUSTOM_SIMPLE_EDGE) -Reducer 59 <- Map 58 (SIMPLE_EDGE), Map 63 (SIMPLE_EDGE) -Reducer 6 <- Reducer 14 (CUSTOM_SIMPLE_EDGE), Reducer 22 (CUSTOM_SIMPLE_EDGE), Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 38 (CUSTOM_SIMPLE_EDGE), Reducer 46 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 54 (CUSTOM_SIMPLE_EDGE), Reducer 62 (CUSTOM_SIMPLE_EDGE) -Reducer 60 <- Map 64 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 61 <- Map 65 (SIMPLE_EDGE), Reducer 60 (SIMPLE_EDGE) -Reducer 62 <- Reducer 61 (CUSTOM_SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 48 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (SORT_PARTITION_EDGE), Vertex 49 (SORT_PARTITION_EDGE) +Vertex 46 <- Vertex 45 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 51 <- Vertex 50 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 56 (SORT_PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE), Vertex 57 (SORT_PARTITION_EDGE) +Vertex 54 <- Vertex 53 (PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 14 (PARTITION_EDGE), Vertex 22 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE), Vertex 38 (PARTITION_EDGE), Vertex 46 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE), Vertex 54 (PARTITION_EDGE), Vertex 62 (PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 64 (SORT_PARTITION_EDGE) +Vertex 61 <- Vertex 60 (SORT_PARTITION_EDGE), Vertex 65 (SORT_PARTITION_EDGE) +Vertex 62 <- Vertex 61 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_218] - Select Operator [SEL_217] (rows=1 width=65) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_347] (rows=1 width=65) - Conds:(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 14 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_209] - Group By Operator [GBY_50] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 13 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_328] (rows=766650239 width=88) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_298] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_327] (rows=696954748 width=88) - Conds:RS_41._col2=RS_42._col0(Inner),Output:["_col1"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_297] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_32] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_326] (rows=633595212 width=88) - Conds:RS_38._col0=RS_39._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_295] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_26] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_296] (rows=14400 width=471) - predicate:((t_hour = 12) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_29] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_210] - Group By Operator [GBY_76] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_75] - Group By Operator [GBY_74] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_331] (rows=766650239 width=88) - Conds:RS_70._col1=RS_71._col0(Inner) - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Select Operator [SEL_63] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_302] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_61] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_330] (rows=696954748 width=88) - Conds:RS_67._col2=RS_68._col0(Inner),Output:["_col1"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_68] - PartitionCols:_col0 - Select Operator [SEL_60] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_301] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_58] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_329] (rows=633595212 width=88) - Conds:RS_64._col0=RS_65._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_299] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_52] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_300] (rows=14400 width=471) - predicate:((t_hour = 11) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_55] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_211] - Group By Operator [GBY_102] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 29 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_101] - Group By Operator [GBY_100] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_334] (rows=766650239 width=88) - Conds:RS_96._col1=RS_97._col0(Inner) - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_306] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_87] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_333] (rows=696954748 width=88) - Conds:RS_93._col2=RS_94._col0(Inner),Output:["_col1"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_305] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_84] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_332] (rows=633595212 width=88) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col2"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_80] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_303] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_78] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Select Operator [SEL_83] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_304] (rows=14400 width=471) - predicate:((t_hour = 11) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_81] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 38 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_212] - Group By Operator [GBY_128] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 37 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_127] - Group By Operator [GBY_126] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_337] (rows=766650239 width=88) - Conds:RS_122._col1=RS_123._col0(Inner) - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_123] - PartitionCols:_col0 - Select Operator [SEL_115] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_310] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_113] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_336] (rows=696954748 width=88) - Conds:RS_119._col2=RS_120._col0(Inner),Output:["_col1"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_120] - PartitionCols:_col0 - Select Operator [SEL_112] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_309] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_110] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_335] (rows=633595212 width=88) - Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_106] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_307] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_104] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Select Operator [SEL_109] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_308] (rows=14400 width=471) - predicate:((t_hour = 10) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_107] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 46 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_213] - Group By Operator [GBY_154] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 45 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_153] - Group By Operator [GBY_152] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_340] (rows=766650239 width=88) - Conds:RS_148._col1=RS_149._col0(Inner) - <-Map 49 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_314] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_139] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_339] (rows=696954748 width=88) - Conds:RS_145._col2=RS_146._col0(Inner),Output:["_col1"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_146] - PartitionCols:_col0 - Select Operator [SEL_138] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_313] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_136] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_145] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_338] (rows=633595212 width=88) - Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col1","_col2"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_142] - PartitionCols:_col0 - Select Operator [SEL_132] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_311] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_130] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col0 - Select Operator [SEL_135] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_312] (rows=14400 width=471) - predicate:((t_hour = 10) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_133] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_208] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_325] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_294] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_324] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_293] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_323] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_291] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_292] (rows=14400 width=471) - predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 54 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_214] - Group By Operator [GBY_180] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 53 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_179] - Group By Operator [GBY_178] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_343] (rows=766650239 width=88) - Conds:RS_174._col1=RS_175._col0(Inner) - <-Map 57 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col0 - Select Operator [SEL_167] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_318] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_165] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_174] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_342] (rows=696954748 width=88) - Conds:RS_171._col2=RS_172._col0(Inner),Output:["_col1"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Select Operator [SEL_164] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_317] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_162] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_341] (rows=633595212 width=88) - Conds:RS_168._col0=RS_169._col0(Inner),Output:["_col1","_col2"] - <-Map 50 [SIMPLE_EDGE] - SHUFFLE [RS_168] - PartitionCols:_col0 - Select Operator [SEL_158] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_315] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_156] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col0 - Select Operator [SEL_161] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_316] (rows=14400 width=471) - predicate:((t_hour = 9) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_159] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 62 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_215] - Group By Operator [GBY_206] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 61 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_205] - Group By Operator [GBY_204] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_346] (rows=766650239 width=88) - Conds:RS_200._col1=RS_201._col0(Inner) - <-Map 65 [SIMPLE_EDGE] - SHUFFLE [RS_201] - PartitionCols:_col0 - Select Operator [SEL_193] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_322] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_191] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_200] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_345] (rows=696954748 width=88) - Conds:RS_197._col2=RS_198._col0(Inner),Output:["_col1"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_198] - PartitionCols:_col0 - Select Operator [SEL_190] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_321] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_188] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_197] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_344] (rows=633595212 width=88) - Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1","_col2"] - <-Map 58 [SIMPLE_EDGE] - SHUFFLE [RS_194] - PartitionCols:_col0 - Select Operator [SEL_184] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_319] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_182] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_195] - PartitionCols:_col0 - Select Operator [SEL_187] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_320] (rows=14400 width=471) - predicate:((t_hour = 9) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_185] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query89.q.out ql/src/test/results/clientpositive/perf/query89.q.out index 7bc8700..4ec3f63 100644 --- ql/src/test/results/clientpositive/perf/query89.q.out +++ ql/src/test/results/clientpositive/perf/query89.q.out @@ -55,95 +55,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=191662559 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Select Operator [SEL_30] (rows=191662559 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_46] (rows=191662559 width=88) - predicate:CASE WHEN ((avg_window_0 <> 0)) THEN (((abs((_col6 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END - Select Operator [SEL_29] (rows=383325119 width=88) - Output:["avg_window_0","_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_28] (rows=383325119 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col2, _col3, _col4"}] - Select Operator [SEL_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col2, _col3, _col4 - Select Operator [SEL_25] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_24] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_22] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col7, _col10, _col12, _col13 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col5","_col6","_col7","_col10","_col12","_col13","_col3"] - Merge Join Operator [MERGEJOIN_53] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col7","_col10","_col12","_col13"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_50] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_52] (rows=696954748 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col7","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36525 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_49] (rows=36525 width=1119) - predicate:((d_year) IN (2000) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_51] (rows=633595212 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col2","_col3","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_48] (rows=231000 width=1436) - predicate:(((i_class) IN ('wallpaper', 'parenting', 'musical') or (i_class) IN ('womens', 'birdal', 'pants')) and ((i_category) IN ('Home', 'Books', 'Electronics') or (i_category) IN ('Shoes', 'Jewelry', 'Men')) and (((i_category) IN ('Home', 'Books', 'Electronics') and (i_class) IN ('wallpaper', 'parenting', 'musical')) or ((i_category) IN ('Shoes', 'Jewelry', 'Men') and (i_class) IN ('womens', 'birdal', 'pants'))) and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query90.q.out ql/src/test/results/clientpositive/perf/query90.q.out index 5ae9fe5..d0e4e46 100644 --- ql/src/test/results/clientpositive/perf/query90.q.out +++ ql/src/test/results/clientpositive/perf/query90.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 @@ -6,143 +6,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (CUSTOM_SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 6 <- Vertex 15 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_59] - Limit [LIM_58] (rows=1 width=17) - Number of rows:100 - Select Operator [SEL_57] (rows=1 width=17) - Output:["_col0"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Select Operator [SEL_55] (rows=1 width=17) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_92] (rows=1 width=17) - Conds:(Inner),Output:["_col0","_col1"] - <-Reducer 15 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Group By Operator [GBY_50] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 14 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_85] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_84] (rows=9600 width=471) - predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null) - TableScan [TS_32] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_89] (rows=158402938 width=135) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col0","_col1"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Select Operator [SEL_28] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - TableScan [TS_26] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_83] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - TableScan [TS_29] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_81] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_80] (rows=9600 width=471) - predicate:(t_hour BETWEEN 6 AND 7 and t_time_sk is not null) - TableScan [TS_6] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_79] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - TableScan [TS_3] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query91.q.out ql/src/test/results/clientpositive/perf/query91.q.out index 1538e8b..fda2585 100644 --- ql/src/test/results/clientpositive/perf/query91.q.out +++ ql/src/test/results/clientpositive/perf/query91.q.out @@ -5,126 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_47] - Select Operator [SEL_46] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_45] - Select Operator [SEL_44] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_43] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_41] (rows=117128008 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col8, _col9, _col10, _col18, _col19 - Select Operator [SEL_40] (rows=117128008 width=860) - Output:["_col8","_col9","_col10","_col18","_col19","_col3"] - Merge Join Operator [MERGEJOIN_81] (rows=117128008 width=860) - Conds:RS_37._col1=RS_38._col2(Inner),Output:["_col3","_col8","_col9","_col10","_col18","_col19"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Select Operator [SEL_30] (rows=106480005 width=860) - Output:["_col2","_col7","_col8"] - Merge Join Operator [MERGEJOIN_80] (rows=106480005 width=860) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col0","_col5","_col6"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_75] (rows=3600 width=107) - predicate:((hd_buy_potential like '0-500%') and hd_demo_sk is not null) - TableScan [TS_18] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=96800003 width=860) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col0","_col2","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_74] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -7) and ca_address_sk is not null) - TableScan [TS_15] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_78] (rows=88000001 width=860) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col0","_col2","_col3","_col5","_col6"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=930900 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_73] (rows=930900 width=385) - predicate:(((cd_education_status = 'Unknown') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'W')) and (((cd_marital_status = 'M') and (cd_education_status = 'Unknown')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree'))) and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_72] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null) - TableScan [TS_9] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_77] (rows=34846646 width=106) - Conds:RS_34._col2=RS_35._col0(Inner),Output:["_col1","_col3","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=60 width=2045) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_71] (rows=60 width=2045) - predicate:cc_call_center_sk is not null - TableScan [TS_6] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_call_center_id","cc_name","cc_manager"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_76] (rows=31678769 width=106) - Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_69] (rows=28798881 width=106) - predicate:(cr_call_center_sk is not null and cr_returned_date_sk is not null and cr_returning_customer_sk is not null) - TableScan [TS_0] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_call_center_sk","cr_net_loss"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_70] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query92.q.out ql/src/test/results/clientpositive/perf/query92.q.out index ca1f417..bf75d3e 100644 --- ql/src/test/results/clientpositive/perf/query92.q.out +++ ql/src/test/results/clientpositive/perf/query92.q.out @@ -5,89 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_37] - Group By Operator [GBY_35] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_34] - Group By Operator [GBY_33] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col0)","sum(_col1)","sum(_col2)"] - Select Operator [SEL_31] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_48] (rows=348477374 width=88) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_46] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,ss,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_43] (rows=8116 width=1119) - predicate:((d_month_seq >= 1206) and (d_month_seq <= 1217) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Group By Operator [GBY_26] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_47] (rows=316788826 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_45] (rows=8116 width=1119) - predicate:((d_month_seq >= 1206) and (d_month_seq <= 1217) and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_44] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query93.q.out ql/src/test/results/clientpositive/perf/query93.q.out index f28ba41..57ea1e0 100644 --- ql/src/test/results/clientpositive/perf/query93.q.out +++ ql/src/test/results/clientpositive/perf/query93.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=316797606 width=88) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_19] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_15] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_34] (rows=633595212 width=88) - Conds:RS_12._col0, _col2=RS_13._col0, _col2(Inner),Output:["_col3","_col7","_col9","_col10"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0, _col2 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_32] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk","ss_customer_sk","ss_ticket_number","ss_quantity","ss_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0, _col2 - Merge Join Operator [MERGEJOIN_33] (rows=63350266 width=77) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_30] (rows=57591150 width=77) - predicate:(sr_reason_sk is not null and sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_reason_sk","sr_ticket_number","sr_return_quantity"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36 width=200) - Output:["_col0"] - Filter Operator [FIL_31] (rows=36 width=200) - predicate:((r_reason_desc = 'Did not like the warranty') and r_reason_sk is not null) - TableScan [TS_3] (rows=72 width=200) - default@reason,reason,Tbl:COMPLETE,Col:NONE,Output:["r_reason_sk","r_reason_desc"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query94.q.out ql/src/test/results/clientpositive/perf/query94.q.out index 836b16b..43b7f98 100644 --- ql/src/test/results/clientpositive/perf/query94.q.out +++ ql/src/test/results/clientpositive/perf/query94.q.out @@ -5,126 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 14 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 10 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_51] - Limit [LIM_50] (rows=1 width=344) - Number of rows:100 - Group By Operator [GBY_48] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Group By Operator [GBY_46] (rows=127554770 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3 - Select Operator [SEL_45] (rows=127554770 width=135) - Output:["_col3","_col4","_col5"] - Filter Operator [FIL_44] (rows=127554770 width=135) - predicate:_col12 is null - Merge Join Operator [MERGEJOIN_85] (rows=255109540 width=135) - Conds:RS_40._col3=RS_41._col0(Left Outer),Output:["_col3","_col4","_col5","_col12"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=14398467 width=92) - Output:["_col0"] - Filter Operator [FIL_79] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_23] (rows=14398467 width=92) - default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=231917759 width=135) - Conds:RS_37._col2=RS_38._col0(Inner),Output:["_col3","_col4","_col5"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_78] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_20] (rows=84 width=1850) - default@web_site,s,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=210834322 width=135) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_77] (rows=20000000 width=1014) - predicate:((ca_state = 'TX') and ca_address_sk is not null) - TableScan [TS_17] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_82] (rows=191667562 width=135) - Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=8116 width=1119) - predicate:(d_date BETWEEN '1999-05-01' AND '1999-07-01' and d_date_sk is not null) - TableScan [TS_14] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=174243235 width=135) - Conds:RS_28._col3=RS_29._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_73] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_web_site_sk is not null and ws_ship_date_sk is not null and ws_order_number is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=158402938 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_13] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_12] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_80] (rows=158402938 width=135) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_6] (rows=144002668 width=135) - default@web_sales,ws3,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_3] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query95.q.out ql/src/test/results/clientpositive/perf/query95.q.out index 9b0d1b2..7812cc2 100644 --- ql/src/test/results/clientpositive/perf/query95.q.out +++ ql/src/test/results/clientpositive/perf/query95.q.out @@ -5,152 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_63] - Group By Operator [GBY_61] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Group By Operator [GBY_59] (rows=510219083 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3 - Merge Join Operator [MERGEJOIN_122] (rows=510219083 width=135) - Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col3","_col4","_col5"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_40] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_115] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_38] (rows=84 width=1850) - default@web_site,s,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_121] (rows=463835520 width=135) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_114] (rows=20000000 width=1014) - predicate:((ca_state = 'GA') and ca_address_sk is not null) - TableScan [TS_35] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_120] (rows=421668646 width=135) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_113] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2002-05-01' AND '2002-06-30' and d_date_sk is not null) - TableScan [TS_32] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=383335125 width=135) - Conds:RS_45._col3=RS_46._col0(Left Semi),RS_45._col3=RS_47._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_107] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_web_site_sk is not null and ws_ship_date_sk is not null and ws_order_number is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_44] (rows=174243235 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_31] (rows=174243235 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_118] (rows=174243235 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=14398467 width=92) - Output:["_col0"] - Filter Operator [FIL_112] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_25] (rows=14398467 width=92) - default@web_returns,wr,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_23] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_117] (rows=158402938 width=135) - Conds:RS_20._col1=RS_21._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_16] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_110] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_14] (rows=144002668 width=135) - default@web_sales,ws4,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_111] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_17] (rows=144002668 width=135) - default@web_sales,ws5,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Group By Operator [GBY_42] (rows=158402938 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_13] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_12] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_116] (rows=158402938 width=135) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_108] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_3] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_109] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_6] (rows=144002668 width=135) - default@web_sales,ws3,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query96.q.out ql/src/test/results/clientpositive/perf/query96.q.out index a6c1f46..ad5bb1d 100644 --- ql/src/test/results/clientpositive/perf/query96.q.out +++ ql/src/test/results/clientpositive/perf/query96.q.out @@ -5,76 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_29] - Limit [LIM_28] (rows=1 width=8) - Number of rows:100 - Select Operator [SEL_27] (rows=1 width=8) - Output:["_col0"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_45] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_42] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_44] (rows=696954748 width=88) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_41] (rows=3600 width=107) - predicate:((hd_dep_count = 5) and hd_demo_sk is not null) - TableScan [TS_6] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_39] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_40] (rows=14400 width=471) - predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query97.q.out ql/src/test/results/clientpositive/perf/query97.q.out index d9c994f..24cc0d0 100644 --- ql/src/test/results/clientpositive/perf/query97.q.out +++ ql/src/test/results/clientpositive/perf/query97.q.out @@ -5,91 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_38] - Limit [LIM_37] (rows=1 width=24) - Number of rows:100 - Group By Operator [GBY_35] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_34] - Group By Operator [GBY_33] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col0)","sum(_col1)","sum(_col2)"] - Select Operator [SEL_31] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_49] (rows=348477374 width=88) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_47] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_44] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Group By Operator [GBY_26] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_48] (rows=316788826 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_45] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query98.q.out ql/src/test/results/clientpositive/perf/query98.q.out index 1bae9be..8a444ee 100644 --- ql/src/test/results/clientpositive/perf/query98.q.out +++ ql/src/test/results/clientpositive/perf/query98.q.out @@ -15,74 +15,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_27] - Select Operator [SEL_26] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_23] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_22] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col3 - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)"],keys:_col10, _col9, _col6, _col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col10","_col9","_col6","_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_37] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_35] (rows=231000 width=1436) - predicate:((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_36] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_33] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_34] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2001-01-12 AND 2001-02-11 00:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index 6602222..6321eab 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -13,30 +13,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 3 <- Union 2 (CONTAINS) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:10 Stage-1 Union 2 - <-Map 1 [CONTAINS] - File Output Operator [FS_7] - Limit [LIM_6] (rows=10/20 width=178) - Number of rows:10 - Select Operator [SEL_1] (rows=500/12 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/13 width=178) - Output:["key","value"] - <-Map 3 [CONTAINS] - File Output Operator [FS_7] - Limit [LIM_6] (rows=10/20 width=178) - Number of rows:10 - Select Operator [SEL_3] (rows=500/12 width=178) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=500/13 width=178) - Output:["key","value"] + <-Vertex 1 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: select key from src PREHOOK: type: QUERY @@ -84,13 +70,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.t"} - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -128,13 +108,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.t"} - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 PREHOOK: query: select key from src limit 10 PREHOOK: type: QUERY @@ -232,23 +206,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=205/309 width=95) - Output:["_col0","_col1"],aggregations:["count(KEY._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - PartitionCols:key - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["key"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select key, count(key) from src group by key)subq PREHOOK: type: QUERY @@ -381,26 +345,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 3 <- Union 2 (CONTAINS) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 Union 2 - <-Map 1 [CONTAINS] - File Output Operator [FS_6] - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - Output:["key","value"] - <-Map 3 [CONTAINS] - File Output Operator [FS_6] - Select Operator [SEL_3] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=500/500 width=178) - Output:["key","value"] + <-Vertex 1 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: select count(*) from (select * from src a union all select * from src b)subq PREHOOK: type: QUERY @@ -432,34 +386,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=1219/1028 width=178) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_15] (rows=1219/1028 width=178) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=87) - default@src,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index 88c68d4..8747e50 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -41,139 +41,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 6 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_56] - Group By Operator [GBY_54] (rows=28/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Select Operator [SEL_49] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_82] (rows=148/61 width=177) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_78] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_40] (rows=500/500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_81] (rows=61/52 width=177) - Conds:RS_43._col1=RS_44._col1(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_37] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_36] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_27] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_25] (rows=25/25 width=175) - Output:["key","value"] - <-Map 16 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_30] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_28] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_80] (rows=148/61 width=177) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_74] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=61/52 width=177) - Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_73] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_12] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_2] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_71] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25/25 width=175) - Output:["key","value"] - <-Map 8 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_72] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500/500 width=178) - Output:["key","value"] + Vertex 7 PREHOOK: query: SELECT x.key, y.value FROM src1 x JOIN src y ON (x.key = y.key) @@ -234,264 +118,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 10 <- Union 2 (CONTAINS) -Map 13 <- Union 14 (CONTAINS) -Map 20 <- Union 14 (CONTAINS) -Map 21 <- Union 16 (CONTAINS) -Map 24 <- Union 25 (CONTAINS) -Map 33 <- Union 25 (CONTAINS) -Map 34 <- Union 27 (CONTAINS) -Map 35 <- Union 29 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) -Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 30 <- Union 29 (SIMPLE_EDGE) -Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 10 <- Union 2 (CONTAINS) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 6 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 14 (CONTAINS) +Vertex 21 <- Union 16 (CONTAINS) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 8 (CONTAINS), Vertex 31 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 33 <- Union 25 (CONTAINS) +Vertex 34 <- Union 27 (CONTAINS) +Vertex 35 <- Union 29 (CONTAINS) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_114] - Group By Operator [GBY_112] (rows=872/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 32 [CONTAINS] - Reduce Output Operator [RS_111] - PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=434/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_162] (rows=434/61 width=177) - Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col2","_col5"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_98] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_161] (rows=178/52 width=86) - Conds:RS_101._col1=RS_102._col1(Inner),Output:["_col2"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_97] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_95] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col1 - Select Operator [SEL_94] (rows=1525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_93] (rows=1525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 29 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] - Reduce Output Operator [RS_92] - PartitionCols:_col1, _col0 - Select Operator [SEL_88] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_86] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 28 [CONTAINS] - Reduce Output Operator [RS_92] - PartitionCols:_col1, _col0 - Select Operator [SEL_85] (rows=1025/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_84] (rows=1025/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 27 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] - Reduce Output Operator [RS_83] - PartitionCols:_col1, _col0 - Select Operator [SEL_79] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_77] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_83] - PartitionCols:_col1, _col0 - Select Operator [SEL_76] (rows=525/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_75] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 24 [CONTAINS] - Reduce Output Operator [RS_74] - PartitionCols:_col1, _col0 - Select Operator [SEL_67] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_65] (rows=25/25 width=175) - Output:["key","value"] - <-Map 33 [CONTAINS] - Reduce Output Operator [RS_74] - PartitionCols:_col1, _col0 - Select Operator [SEL_70] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_68] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 7 [CONTAINS] - Reduce Output Operator [RS_111] - PartitionCols:_col0, _col1 - Group By Operator [GBY_63] (rows=438/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_62] - PartitionCols:_col0, _col1 - Select Operator [SEL_58] (rows=290/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_160] (rows=290/61 width=177) - Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_49] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_159] (rows=119/52 width=86) - Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_46] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=1025/319 width=178) - Output:["_col1"] - Group By Operator [GBY_44] (rows=1025/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 16 [SIMPLE_EDGE] - <-Map 21 [CONTAINS] - Reduce Output Operator [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_39] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_37] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_36] (rows=525/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_27] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_146] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_25] (rows=25/25 width=175) - Output:["key","value"] - <-Map 20 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_30] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_147] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_28] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_62] - PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_158] (rows=148/61 width=177) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_157] (rows=61/52 width=86) - Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_144] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_12] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_2] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_142] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25/25 width=175) - Output:["key","value"] - <-Map 10 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_143] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500/500 width=178) - Output:["key","value"] + Vertex 9 PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -647,22 +302,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value PREHOOK: type: QUERY @@ -683,42 +323,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key PREHOOK: type: QUERY @@ -746,22 +357,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value PREHOOK: type: QUERY @@ -786,42 +382,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab2,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key UNION ALL @@ -1126,6 +693,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +<<<<<<< HEAD Map 1 <- Union 2 (CONTAINS) Map 13 <- Union 10 (CONTAINS) Map 14 <- Union 10 (CONTAINS) @@ -1141,6 +709,23 @@ Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 22 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) +======= +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 10 (CONTAINS) +Vertex 13 <- Union 10 (CONTAINS) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 17 <- Union 18 (CONTAINS) +Vertex 19 <- Union 18 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 18 (CONTAINS) +Vertex 21 <- Union 18 (CONTAINS) +Vertex 22 <- Union 18 (CONTAINS) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 10 (CONTAINS) +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator @@ -1150,6 +735,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 +<<<<<<< HEAD Union 5 <-Reducer 12 [CONTAINS] File Output Operator [FS_79] @@ -1343,6 +929,12 @@ Stage-5 File Output Operator [FS_83] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_20] +======= + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 3 [CONTAINS] +>>>>>>> renamed edges Stage-6 Stats-Aggr Operator Stage-1 @@ -1431,6 +1023,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +<<<<<<< HEAD Map 1 <- Union 2 (CONTAINS) Map 10 <- Union 2 (CONTAINS) Map 13 <- Union 14 (CONTAINS) @@ -1454,6 +1047,31 @@ Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) +======= +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 14 (CONTAINS) +Vertex 20 <- Union 16 (CONTAINS) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Union 7 (CONTAINS), Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 25 (CONTAINS) +Vertex 33 <- Union 27 (CONTAINS) +Vertex 34 <- Union 29 (CONTAINS) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator @@ -1463,6 +1081,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 +<<<<<<< HEAD Reducer 9 File Output Operator [FS_115] table:{"name:":"default.a"} @@ -1700,6 +1319,9 @@ Stage-5 File Output Operator [FS_119] table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_112] +======= + Vertex 8 +>>>>>>> renamed edges Stage-6 Stats-Aggr Operator Stage-1 @@ -1777,6 +1399,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 5 File Output Operator [FS_18] table:{"name:":"default.dest1"} @@ -1812,6 +1435,9 @@ Stage-4 Group By Operator [GBY_21] (rows=501/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_12] +======= + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 @@ -1846,33 +1472,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Unique_join]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=594/4122 width=260) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_9] (rows=594/4122 width=260) - Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=500/500 width=87) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:key - TableScan [TS_1] (rows=25/25 width=86) - default@src1,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=2000/2000 width=87) - default@srcpart,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: SELECT TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) @@ -1905,32 +1511,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=1219/1028 width=178) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=1219/1028 width=178) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_1] (rows=500/500 width=87) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: FROM ( select key, value from ( @@ -2005,6 +1592,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 4 File Output Operator [FS_17] table:{"name:":"default.dest1"} @@ -2052,6 +1640,10 @@ Stage-4 Group By Operator [GBY_20] (rows=1001/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] +======= + Vertex 4 + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 @@ -2119,6 +1711,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 4 File Output Operator [FS_14] table:{"name:":"default.dest1"} @@ -2156,6 +1749,10 @@ Stage-4 Group By Operator [GBY_17] (rows=501/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] +======= + Vertex 4 + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 9f1a401..7c5df3c 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -227,9 +227,7 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -345,13 +343,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.src_autho_test"} - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -599,24 +591,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 - File Output Operator [FS_5] - Limit [LIM_4] (rows=5/5 width=178) - Number of rows:5 - Select Operator [SEL_3] (rows=500/5 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE @@ -660,15 +641,7 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 - File Output Operator [FS_3] - table:{"name:":"default.orc_merge5"} - Select Operator [SEL_2] (rows=306/3 width=268) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_4] (rows=306/3 width=268) - predicate:(userid <= 13) - TableScan [TS_0] (rows=919/15000 width=268) - default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] + Vertex 1 Stage-4(CONDITIONAL) File Merge Please refer to the previous Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) diff --git ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out index 14535f6..715bcbf 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out @@ -35,39 +35,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=2166/10 width=620) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_17] (rows=2166/10 width=620) - Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365/10 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365/10 width=251) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=251) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1019/10 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1019/10 width=251) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=251) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -222,45 +197,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_15] - Select Operator [SEL_14] (rows=615/5 width=12) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_13] - Group By Operator [GBY_11] (rows=615/5 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_20] (rows=2166/10 width=4) - Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365/10 width=5) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=1365/10 width=5) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=5) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019/10 width=8) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1019/10 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -324,39 +269,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1501/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_17] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -511,45 +431,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_15] - Select Operator [SEL_14] (rows=750/5 width=215) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_13] - Group By Operator [GBY_11] (rows=750/5 width=215) - Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Map Join Operator [MAPJOIN_20] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index ee9affb..e4a623f 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -29,9 +29,7 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -279,7 +277,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -289,20 +287,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 - File Output Operator [FS_8] - table:{"name:":"default.acid_uami"} - Select Operator [SEL_4] (rows=8/2 width=302) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=8/2 width=302) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_9] (rows=8/2 width=226) - predicate:((de = 109.23) or (de = 119.23)) - TableScan [TS_0] (rows=8/4 width=226) - default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"] + Vertex 2 PREHOOK: query: select * from acid_uami order by de PREHOOK: type: QUERY @@ -392,7 +377,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze delete from acid_dot where cint < -1070551679 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -402,20 +387,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 - File Output Operator [FS_7] - table:{"name:":"default.acid_dot"} - Select Operator [SEL_4] (rows=31436/8 width=4) - Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=31436/8 width=4) - Output:["_col0"] - Filter Operator [FIL_8] (rows=31436/8 width=4) - predicate:(cint < -1070551679) - TableScan [TS_0] (rows=94309/12288 width=4) - default@acid_dot,acid_dot, ACID table,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 2 PREHOOK: query: select count(*) from acid_dot PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/tez/explainuser_3.q.out ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 17c9ec3..dd9e607 100644 --- ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -271,13 +271,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.src_autho_test"} - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -451,24 +445,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 vectorized - File Output Operator [FS_8] - Limit [LIM_7] (rows=5 width=178) - Number of rows:5 - Select Operator [SEL_6] (rows=500 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE @@ -504,15 +487,7 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 vectorized - File Output Operator [FS_10] - table:{"name:":"default.orc_merge5"} - Select Operator [SEL_9] (rows=306 width=268) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_8] (rows=306 width=268) - predicate:(userid <= 13) - TableScan [TS_0] (rows=919 width=268) - default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] + Vertex 1 Stage-4(CONDITIONAL) File Merge Please refer to the previous Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) diff --git ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out index f3a5dc3..01c7379 100644 --- ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -66,7 +66,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -83,20 +97,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -171,7 +171,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -183,7 +183,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -200,20 +214,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -286,7 +286,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -297,7 +297,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -314,20 +328,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -398,7 +398,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -410,7 +410,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -427,20 +441,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -508,7 +508,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -519,21 +519,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -547,6 +533,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: cd + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -614,7 +614,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -626,21 +626,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -654,6 +640,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: cd + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -756,7 +756,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -767,7 +767,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -784,20 +798,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -872,7 +872,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -884,7 +884,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -901,20 +915,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -989,7 +989,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1000,7 +1000,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1017,20 +1031,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1105,7 +1105,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -1117,7 +1117,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1134,20 +1148,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1206,7 +1206,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1226,10 +1226,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1249,7 +1249,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1263,7 +1263,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1287,7 +1287,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1402,7 +1402,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1422,10 +1422,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1445,7 +1445,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1459,7 +1459,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1483,7 +1483,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out index 6f5a3a9..cd42d3a 100644 --- ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out @@ -52,7 +52,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -69,8 +69,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -81,20 +81,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -108,6 +95,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -176,7 +176,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -193,8 +193,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -206,20 +206,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -233,6 +220,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -312,7 +312,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -331,9 +331,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -344,7 +344,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -357,7 +371,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -370,20 +384,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -456,7 +456,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -475,9 +475,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -489,7 +489,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -502,7 +516,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -515,20 +529,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -619,7 +619,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -632,7 +632,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -649,8 +649,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -661,7 +661,38 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -674,7 +705,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -687,7 +718,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -704,8 +735,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -716,38 +747,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -764,8 +764,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -854,7 +852,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -867,7 +865,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -884,8 +882,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -897,7 +895,38 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -910,7 +939,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -923,7 +952,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -940,8 +969,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -953,38 +982,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1001,8 +999,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -1100,7 +1096,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: value (type: string) - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1118,8 +1114,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1130,8 +1126,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1142,7 +1138,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1155,7 +1165,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1168,7 +1178,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1181,20 +1191,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1284,7 +1280,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: value (type: string) - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1302,8 +1298,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Map Join Operator @@ -1315,8 +1311,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -1328,7 +1324,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1341,7 +1351,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1354,7 +1364,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1367,20 +1377,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out index 8006a90..4165c57 100644 --- ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out +++ ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out @@ -102,36 +102,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_21] (rows=2 width=431) - Conds:RS_23._col2=RS_28._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_23] - PartitionCols:_col2 - Select Operator [SEL_22] (rows=2 width=134) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=2 width=236) - default@char_tbl1,c1,Tbl:COMPLETE,Col:PARTIAL,Output:["name","age"] - Dynamic Partitioning Event Operator [EVENT_26] (rows=1 width=134) - Group By Operator [GBY_25] (rows=1 width=134) - Output:["_col0"],keys:_col0 - Select Operator [SEL_24] (rows=2 width=134) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_22] - <-Map 3 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_28] - PartitionCols:_col2 - Select Operator [SEL_27] (rows=2 width=89) - Output:["_col0","_col1","_col2"] - TableScan [TS_3] (rows=2 width=190) - default@char_tbl2,c2,Tbl:COMPLETE,Col:PARTIAL,Output:["name","age"] + Vertex 2 PREHOOK: query: select c1.name, c1.age, c1.gpa, c2.name, c2.age, c2.gpa from char_tbl1 c1 join char_tbl2 c2 on (c1.gpa = c2.gpa) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out index 4535e66..624ace1 100644 --- ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out +++ ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out @@ -40,10 +40,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: non_string_part @@ -62,7 +62,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: tinyint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -119,10 +119,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: non_string_part @@ -140,7 +140,7 @@ STAGE PLANS: Statistics: Num rows: 1024 Data size: 121205 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/tez/vectorization_div0.q.out ql/src/test/results/clientpositive/tez/vectorization_div0.q.out index 3c017e6..a4431b0 100644 --- ql/src/test/results/clientpositive/tez/vectorization_div0.q.out +++ ql/src/test/results/clientpositive/tez/vectorization_div0.q.out @@ -146,10 +146,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -168,7 +168,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: decimal(22,21)) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -319,10 +319,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -341,7 +341,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: double), _col4 (type: double), _col5 (type: double) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/tez/vectorization_limit.q.out ql/src/test/results/clientpositive/tez/vectorization_limit.q.out index dfb0102..8d50a8c 100644 --- ql/src/test/results/clientpositive/tez/vectorization_limit.q.out +++ ql/src/test/results/clientpositive/tez/vectorization_limit.q.out @@ -53,10 +53,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -75,7 +75,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 value expressions: _col2 (type: smallint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -142,10 +142,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -168,7 +168,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: struct) Execution mode: vectorized - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0) @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -260,7 +260,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -328,10 +328,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -351,7 +351,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -449,11 +449,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -478,7 +478,7 @@ STAGE PLANS: Statistics: Num rows: 3185 Data size: 44512 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -492,7 +492,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 3185 Data size: 44512 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: vectorized Reduce Operator Tree: Select Operator